{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inspired by: http://blog.varunajayasiri.com/numpy_lstm.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy import ndarray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Dict, List, Tuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from IPython import display\n",
    "plt.style.use('seaborn-white')\n",
    "%matplotlib inline\n",
    "\n",
    "from copy import deepcopy\n",
    "from collections import deque"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lincoln.utils.np_utils import assert_same_shape\n",
    "from scipy.special import logsumexp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x: ndarray):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "\n",
    "def dsigmoid(x: ndarray):\n",
    "    return sigmoid(x) * (1 - sigmoid(x))\n",
    "\n",
    "\n",
    "def tanh(x: ndarray):\n",
    "    return np.tanh(x)\n",
    "\n",
    "\n",
    "def dtanh(x: ndarray):\n",
    "    return 1 - np.tanh(x) * np.tanh(x)\n",
    "\n",
    "\n",
    "def softmax(x, axis=None):\n",
    "    return np.exp(x - logsumexp(x, axis=axis, keepdims=True))\n",
    "\n",
    "\n",
    "def batch_softmax(input_array: ndarray):\n",
    "    out = []\n",
    "    for row in input_array:\n",
    "        out.append(softmax(row, axis=1))\n",
    "    return np.stack(out)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `RNNOptimizer`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNNOptimizer(object):\n",
    "    def __init__(self,\n",
    "                 lr: float = 0.01,\n",
    "                 gradient_clipping: bool = True) -> None:\n",
    "        self.lr = lr\n",
    "        self.gradient_clipping = gradient_clipping\n",
    "        self.first = True\n",
    "\n",
    "    def step(self) -> None:\n",
    "\n",
    "        for layer in self.model.layers:\n",
    "            for key in layer.params.keys():\n",
    "\n",
    "                if self.gradient_clipping:\n",
    "                    np.clip(layer.params[key]['deriv'], -2, 2, layer.params[key]['deriv'])\n",
    "\n",
    "                self._update_rule(param=layer.params[key]['value'],\n",
    "                                  grad=layer.params[key]['deriv'])\n",
    "\n",
    "    def _update_rule(self, **kwargs) -> None:\n",
    "        raise NotImplementedError()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `SGD` and `AdaGrad`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SGD(RNNOptimizer):\n",
    "    def __init__(self,\n",
    "                 lr: float = 0.01,\n",
    "                 gradient_clipping: bool = True) -> None:\n",
    "        super().__init__(lr, gradient_clipping)\n",
    "\n",
    "    def _update_rule(self, **kwargs) -> None:\n",
    "\n",
    "        update = self.lr*kwargs['grad']\n",
    "        kwargs['param'] -= update\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AdaGrad(RNNOptimizer):\n",
    "    def __init__(self,\n",
    "                 lr: float = 0.01,\n",
    "                gradient_clipping: bool = True) -> None:\n",
    "        super().__init__(lr, gradient_clipping)\n",
    "        self.eps = 1e-7\n",
    "\n",
    "    def step(self) -> None:\n",
    "        if self.first:\n",
    "            self.sum_squares = {}\n",
    "            for i, layer in enumerate(self.model.layers):\n",
    "                self.sum_squares[i] = {}\n",
    "                for key in layer.params.keys():\n",
    "                    self.sum_squares[i][key] = np.zeros_like(layer.params[key]['value'])\n",
    "            \n",
    "            self.first = False\n",
    "\n",
    "        for i, layer in enumerate(self.model.layers):\n",
    "            for key in layer.params.keys():\n",
    "                \n",
    "                if self.gradient_clipping:\n",
    "                    np.clip(layer.params[key]['deriv'], -2, 2, layer.params[key]['deriv'])\n",
    "                \n",
    "                self._update_rule(param=layer.params[key]['value'],\n",
    "                                  grad=layer.params[key]['deriv'],\n",
    "                                  sum_square=self.sum_squares[i][key])\n",
    "\n",
    "    def _update_rule(self, **kwargs) -> None:\n",
    "\n",
    "            # Update running sum of squares\n",
    "            kwargs['sum_square'] += (self.eps +\n",
    "                                     np.power(kwargs['grad'], 2))\n",
    "\n",
    "            # Scale learning rate by running sum of squareds=5\n",
    "            lr = np.divide(self.lr, np.sqrt(kwargs['sum_square']))\n",
    "\n",
    "            # Use this to update parameters\n",
    "            kwargs['param'] -= lr * kwargs['grad']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `Loss`es"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Loss(object):\n",
    "\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def forward(self,\n",
    "                prediction: ndarray,\n",
    "                target: ndarray) -> float:\n",
    "\n",
    "        assert_same_shape(prediction, target)\n",
    "\n",
    "        self.prediction = prediction\n",
    "        self.target = target\n",
    "\n",
    "        self.output = self._output()\n",
    "\n",
    "        return self.output\n",
    "    \n",
    "    def backward(self) -> ndarray:\n",
    "\n",
    "        self.input_grad = self._input_grad()\n",
    "\n",
    "        assert_same_shape(self.prediction, self.input_grad)\n",
    "\n",
    "        return self.input_grad\n",
    "\n",
    "    def _output(self) -> float:\n",
    "        raise NotImplementedError()\n",
    "\n",
    "    def _input_grad(self) -> ndarray:\n",
    "        raise NotImplementedError()\n",
    "\n",
    "        \n",
    "class SoftmaxCrossEntropy(Loss):\n",
    "    def __init__(self, eps: float=1e-9) -> None:\n",
    "        super().__init__()\n",
    "        self.eps = eps\n",
    "        self.single_class = False\n",
    "\n",
    "    def _output(self) -> float:\n",
    "\n",
    "        out = []\n",
    "        for row in self.prediction:\n",
    "            out.append(softmax(row, axis=1))\n",
    "        softmax_preds = np.stack(out)\n",
    "\n",
    "        # clipping the softmax output to prevent numeric instability\n",
    "        self.softmax_preds = np.clip(softmax_preds, self.eps, 1 - self.eps)\n",
    "\n",
    "        # actual loss computation\n",
    "        softmax_cross_entropy_loss = -1.0 * self.target * np.log(self.softmax_preds) - \\\n",
    "            (1.0 - self.target) * np.log(1 - self.softmax_preds)\n",
    "\n",
    "        return np.sum(softmax_cross_entropy_loss)\n",
    "\n",
    "    def _input_grad(self) -> np.ndarray:\n",
    "\n",
    "        return self.softmax_preds - self.target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNNs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `RNNNode`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNNNode(object):\n",
    "\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def forward(self,\n",
    "                x_in: ndarray, \n",
    "                H_in: ndarray,\n",
    "                params_dict: Dict[str, Dict[str, ndarray]]\n",
    "                ) -> Tuple[ndarray]:\n",
    "        '''\n",
    "        param x: numpy array of shape (batch_size, vocab_size)\n",
    "        param H_prev: numpy array of shape (batch_size, hidden_size)\n",
    "        return self.x_out: numpy array of shape (batch_size, vocab_size)\n",
    "        return self.H: numpy array of shape (batch_size, hidden_size)\n",
    "        '''\n",
    "        self.X_in = x_in\n",
    "        self.H_in = H_in\n",
    "    \n",
    "        self.Z = np.column_stack((x_in, H_in))\n",
    "        \n",
    "        self.H_int = np.dot(self.Z, params_dict['W_f']['value']) \\\n",
    "                                    + params_dict['B_f']['value']\n",
    "        \n",
    "        self.H_out = tanh(self.H_int)\n",
    "\n",
    "        self.X_out = np.dot(self.H_out, params_dict['W_v']['value']) \\\n",
    "                                        + params_dict['B_v']['value']\n",
    "        \n",
    "        return self.X_out, self.H_out\n",
    "\n",
    "\n",
    "    def backward(self, \n",
    "                 X_out_grad: ndarray, \n",
    "                 H_out_grad: ndarray,\n",
    "                 params_dict: Dict[str, Dict[str, ndarray]]) -> Tuple[ndarray]:\n",
    "        '''\n",
    "        param x_out_grad: numpy array of shape (batch_size, vocab_size)\n",
    "        param h_out_grad: numpy array of shape (batch_size, hidden_size)\n",
    "        param RNN_Params: RNN_Params object\n",
    "        return x_in_grad: numpy array of shape (batch_size, vocab_size)\n",
    "        return h_in_grad: numpy array of shape (batch_size, hidden_size)\n",
    "        '''\n",
    "        \n",
    "        assert_same_shape(X_out_grad, self.X_out)\n",
    "        assert_same_shape(H_out_grad, self.H_out)\n",
    "\n",
    "        params_dict['B_v']['deriv'] += X_out_grad.sum(axis=0)\n",
    "        params_dict['W_v']['deriv'] += np.dot(self.H_out.T, X_out_grad)\n",
    "        \n",
    "        dh = np.dot(X_out_grad, params_dict['W_v']['value'].T)\n",
    "        dh += H_out_grad\n",
    "        \n",
    "        dH_int = dh * dtanh(self.H_int)\n",
    "        \n",
    "        params_dict['B_f']['deriv'] += dH_int.sum(axis=0)\n",
    "        params_dict['W_f']['deriv'] += np.dot(self.Z.T, dH_int)     \n",
    "        \n",
    "        dz = np.dot(dH_int, params_dict['W_f']['value'].T)\n",
    "\n",
    "        X_in_grad = dz[:, :self.X_in.shape[1]]\n",
    "        H_in_grad = dz[:, self.X_in.shape[1]:]\n",
    "\n",
    "        assert_same_shape(X_out_grad, self.X_out)\n",
    "        assert_same_shape(H_out_grad, self.H_out)\n",
    "        \n",
    "        return X_in_grad, H_in_grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `RNNLayer`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNNLayer(object):\n",
    "\n",
    "    def __init__(self,\n",
    "                 hidden_size: int,\n",
    "                 output_size: int,\n",
    "                 weight_scale: float = None):\n",
    "        '''\n",
    "        param sequence_length: int - length of sequence being passed through the network\n",
    "        param vocab_size: int - the number of characters in the vocabulary of which we are predicting the next\n",
    "        character.\n",
    "        param hidden_size: int - the number of \"hidden neurons\" in the LSTM_Layer of which this node is a part.\n",
    "        param learning_rate: float - the learning rate\n",
    "        '''\n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        self.weight_scale = weight_scale\n",
    "        self.start_H = np.zeros((1, hidden_size))\n",
    "        self.first = True\n",
    "\n",
    "\n",
    "    def _init_params(self,\n",
    "                     input_: ndarray):\n",
    "        \n",
    "        self.vocab_size = input_.shape[2]\n",
    "        \n",
    "        if not self.weight_scale:\n",
    "            self.weight_scale = 2 / (self.vocab_size + self.output_size)\n",
    "        \n",
    "        self.params = {}\n",
    "        self.params['W_f'] = {}\n",
    "        self.params['B_f'] = {}\n",
    "        self.params['W_v'] = {}\n",
    "        self.params['B_v'] = {}\n",
    "        \n",
    "        self.params['W_f']['value'] = np.random.normal(loc = 0.0,\n",
    "                                                      scale=self.weight_scale,\n",
    "                                                      size=(self.hidden_size + self.vocab_size, self.hidden_size))\n",
    "        self.params['B_f']['value'] = np.random.normal(loc = 0.0,\n",
    "                                                      scale=self.weight_scale,\n",
    "                                                      size=(1, self.hidden_size))\n",
    "        self.params['W_v']['value'] = np.random.normal(loc=0.0,\n",
    "                                                      scale=self.weight_scale,\n",
    "                                                      size=(self.hidden_size, self.output_size))\n",
    "        self.params['B_v']['value'] = np.random.normal(loc=0.0,\n",
    "                                                      scale=self.weight_scale,\n",
    "                                                      size=(1, self.output_size))    \n",
    "        \n",
    "        self.params['W_f']['deriv'] = np.zeros_like(self.params['W_f']['value'])\n",
    "        self.params['B_f']['deriv'] = np.zeros_like(self.params['B_f']['value'])\n",
    "        self.params['W_v']['deriv'] = np.zeros_like(self.params['W_v']['value'])\n",
    "        self.params['B_v']['deriv'] = np.zeros_like(self.params['B_v']['value'])\n",
    "        \n",
    "        self.cells = [RNNNode() for x in range(input_.shape[1])]\n",
    "\n",
    "    \n",
    "    def _clear_gradients(self):\n",
    "        for key in self.params.keys():\n",
    "            self.params[key]['deriv'] = np.zeros_like(self.params[key]['deriv'])\n",
    "        \n",
    "\n",
    "    def forward(self, x_seq_in: ndarray):\n",
    "        '''\n",
    "        param x_seq_in: numpy array of shape (batch_size, sequence_length, vocab_size)\n",
    "        return x_seq_out: numpy array of shape (batch_size, sequence_length, output_size)\n",
    "        '''\n",
    "        if self.first:\n",
    "            self._init_params(x_seq_in)\n",
    "            self.first=False\n",
    "        \n",
    "        batch_size = x_seq_in.shape[0]\n",
    "        \n",
    "        H_in = np.copy(self.start_H)\n",
    "        \n",
    "        H_in = np.repeat(H_in, batch_size, axis=0)\n",
    "\n",
    "        sequence_length = x_seq_in.shape[1]\n",
    "        \n",
    "        x_seq_out = np.zeros((batch_size, sequence_length, self.output_size))\n",
    "        \n",
    "        for t in range(sequence_length):\n",
    "\n",
    "            x_in = x_seq_in[:, t, :]\n",
    "            \n",
    "            y_out, H_in = self.cells[t].forward(x_in, H_in, self.params)\n",
    "      \n",
    "            x_seq_out[:, t, :] = y_out\n",
    "    \n",
    "        self.start_H = H_in.mean(axis=0, keepdims=True)\n",
    "        \n",
    "        return x_seq_out\n",
    "\n",
    "\n",
    "    def backward(self, x_seq_out_grad: ndarray):\n",
    "        '''\n",
    "        param loss_grad: numpy array of shape (batch_size, sequence_length, vocab_size)\n",
    "        return loss_grad_out: numpy array of shape (batch_size, sequence_length, vocab_size)\n",
    "        '''\n",
    "        batch_size = x_seq_out_grad.shape[0]\n",
    "        \n",
    "        h_in_grad = np.zeros((batch_size, self.hidden_size))\n",
    "        \n",
    "        sequence_length = x_seq_out_grad.shape[1]\n",
    "        \n",
    "        x_seq_in_grad = np.zeros((batch_size, sequence_length, self.vocab_size))\n",
    "        \n",
    "        for t in reversed(range(sequence_length)):\n",
    "            \n",
    "            x_out_grad = x_seq_out_grad[:, t, :]\n",
    "\n",
    "            grad_out, h_in_grad = \\\n",
    "                self.cells[t].backward(x_out_grad, h_in_grad, self.params)\n",
    "        \n",
    "            x_seq_in_grad[:, t, :] = grad_out\n",
    "        \n",
    "        return x_seq_in_grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `RNNModel`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNNModel(object):\n",
    "    '''\n",
    "    The Model class that takes in inputs and targets and actually trains the network and calculates the loss.\n",
    "    '''\n",
    "    def __init__(self, \n",
    "                 layers: List[RNNLayer],\n",
    "                 sequence_length: int, \n",
    "                 vocab_size: int, \n",
    "                 loss: Loss):\n",
    "        '''\n",
    "        param num_layers: int - the number of layers in the network\n",
    "        param sequence_length: int - length of sequence being passed through the network\n",
    "        param vocab_size: int - the number of characters in the vocabulary of which we are predicting the next\n",
    "        character.\n",
    "        param hidden_size: int - the number of \"hidden neurons\" in the each layer of the network.\n",
    "        '''\n",
    "        self.layers = layers\n",
    "        self.vocab_size = vocab_size\n",
    "        self.sequence_length = sequence_length\n",
    "        self.loss = loss\n",
    "        for layer in self.layers:\n",
    "            setattr(layer, 'sequence_length', sequence_length)\n",
    "\n",
    "        \n",
    "    def forward(self, \n",
    "                x_batch: ndarray):\n",
    "        '''\n",
    "        param inputs: list of integers - a list of indices of characters being passed in as the \n",
    "        input sequence of the network.\n",
    "        returns x_batch_in: numpy array of shape (batch_size, sequence_length, vocab_size)\n",
    "        '''       \n",
    "        \n",
    "        for layer in self.layers:\n",
    "\n",
    "            x_batch = layer.forward(x_batch)\n",
    "                \n",
    "        return x_batch\n",
    "        \n",
    "    def backward(self, \n",
    "                 loss_grad: ndarray):\n",
    "        '''\n",
    "        param loss_grad: numpy array with shape (batch_size, sequence_length, vocab_size)\n",
    "        returns loss: float, representing mean squared error loss\n",
    "        '''\n",
    "\n",
    "        for layer in reversed(self.layers):\n",
    "\n",
    "            loss_grad = layer.backward(loss_grad)\n",
    "            \n",
    "        return loss_grad\n",
    "                \n",
    "    def single_step(self, \n",
    "                    x_batch: ndarray, \n",
    "                    y_batch: ndarray):\n",
    "        '''\n",
    "        The step that does it all:\n",
    "        1. Forward pass & softmax\n",
    "        2. Compute loss and loss gradient\n",
    "        3. Backward pass\n",
    "        4. Update parameters\n",
    "        param inputs: array of length sequence_length that represents the character indices of the inputs to\n",
    "        the network\n",
    "        param targets: array of length sequence_length that represents the character indices of the targets\n",
    "        of the network \n",
    "        return loss\n",
    "        '''  \n",
    "        \n",
    "        x_batch_out = self.forward(x_batch)\n",
    "        \n",
    "        loss = self.loss.forward(x_batch_out, y_batch)\n",
    "        \n",
    "        loss_grad = self.loss.backward()\n",
    "        \n",
    "        for layer in self.layers:\n",
    "            layer._clear_gradients()\n",
    "        \n",
    "        self.backward(loss_grad)\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `RNNTrainer`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNNTrainer:\n",
    "    '''\n",
    "    Takes in a text file and a model, and starts generating characters.\n",
    "    '''\n",
    "    def __init__(self, \n",
    "                 text_file: str, \n",
    "                 model: RNNModel,\n",
    "                 optim: RNNOptimizer,\n",
    "                 batch_size: int = 32):\n",
    "        self.data = open(text_file, 'r').read()\n",
    "        self.model = model\n",
    "        self.chars = list(set(self.data))\n",
    "        self.vocab_size = len(self.chars)\n",
    "        self.char_to_idx = {ch:i for i,ch in enumerate(self.chars)}\n",
    "        self.idx_to_char = {i:ch for i,ch in enumerate(self.chars)}\n",
    "        self.sequence_length = self.model.sequence_length\n",
    "        self.batch_size = batch_size\n",
    "        self.optim = optim\n",
    "        setattr(self.optim, 'model', self.model)\n",
    "    \n",
    "\n",
    "    def _generate_inputs_targets(self, \n",
    "                                 start_pos: int):\n",
    "        \n",
    "        inputs_indices = np.zeros((self.batch_size, self.sequence_length), dtype=int)\n",
    "        targets_indices = np.zeros((self.batch_size, self.sequence_length), dtype=int)\n",
    "        \n",
    "        for i in range(self.batch_size):\n",
    "            \n",
    "            inputs_indices[i, :] = np.array([self.char_to_idx[ch] \n",
    "                            for ch in self.data[start_pos + i: start_pos + self.sequence_length  + i]])\n",
    "            targets_indices[i, :] = np.array([self.char_to_idx[ch] \n",
    "                         for ch in self.data[start_pos + 1 + i: start_pos + self.sequence_length + 1 + i]])\n",
    "\n",
    "        return inputs_indices, targets_indices\n",
    "\n",
    "\n",
    "    def _generate_one_hot_array(self, \n",
    "                                indices: ndarray):\n",
    "        '''\n",
    "        param indices: numpy array of shape (batch_size, sequence_length)\n",
    "        return batch - numpy array of shape (batch_size, sequence_length, vocab_size)\n",
    "        ''' \n",
    "        batch = []\n",
    "        for seq in indices:\n",
    "            \n",
    "            one_hot_sequence = np.zeros((self.sequence_length, self.vocab_size))\n",
    "            \n",
    "            for i in range(self.sequence_length):\n",
    "                one_hot_sequence[i, seq[i]] = 1.0\n",
    "\n",
    "            batch.append(one_hot_sequence) \n",
    "\n",
    "        return np.stack(batch)\n",
    "\n",
    "\n",
    "    def sample_output(self, \n",
    "                      input_char: int, \n",
    "                      sample_length: int):\n",
    "        '''\n",
    "        Generates a sample output using the current trained model, one character at a time.\n",
    "        param input_char: int - index of the character to use to start generating a sequence\n",
    "        param sample_length: int - the length of the sample output to generate\n",
    "        return txt: string - a string of length sample_length representing the sample output\n",
    "        '''\n",
    "        indices = []\n",
    "        \n",
    "        sample_model = deepcopy(self.model)\n",
    "        \n",
    "        for i in range(sample_length):\n",
    "            input_char_batch = np.zeros((1, 1, self.vocab_size))\n",
    "            \n",
    "            input_char_batch[0, 0, input_char] = 1.0\n",
    "            \n",
    "            x_batch_out = sample_model.forward(input_char_batch)\n",
    "            \n",
    "            x_softmax = batch_softmax(x_batch_out)\n",
    "            \n",
    "            input_char = np.random.choice(range(self.vocab_size), p=x_softmax.ravel())\n",
    "            \n",
    "            indices.append(input_char)\n",
    "            \n",
    "        txt = ''.join(self.idx_to_char[idx] for idx in indices)\n",
    "        return txt\n",
    "\n",
    "    def train(self, \n",
    "              num_iterations: int, \n",
    "              sample_every: int=100):\n",
    "        '''\n",
    "        Trains the \"character generator\" for a number of iterations. \n",
    "        Each \"iteration\" feeds a batch size of 1 through the neural network.\n",
    "        Continues until num_iterations is reached. Displays sample text generated using the latest version.\n",
    "        '''\n",
    "        plot_iter = np.zeros((0))\n",
    "        plot_loss = np.zeros((0))\n",
    "        \n",
    "        num_iter = 0\n",
    "        start_pos = 0\n",
    "        \n",
    "        moving_average = deque(maxlen=100)\n",
    "        while num_iter < num_iterations:\n",
    "            \n",
    "            if start_pos + self.sequence_length + self.batch_size + 1 > len(self.data):\n",
    "                start_pos = 0\n",
    "            \n",
    "            ## Update the model\n",
    "            inputs_indices, targets_indices = self._generate_inputs_targets(start_pos)\n",
    "\n",
    "            inputs_batch, targets_batch = \\\n",
    "                self._generate_one_hot_array(inputs_indices), self._generate_one_hot_array(targets_indices)\n",
    "            \n",
    "            loss = self.model.single_step(inputs_batch, targets_batch)\n",
    "            self.optim.step()\n",
    "            \n",
    "            moving_average.append(loss)\n",
    "            ma_loss = np.mean(moving_average)\n",
    "            \n",
    "            start_pos += self.batch_size\n",
    "            \n",
    "            plot_iter = np.append(plot_iter, [num_iter])\n",
    "            plot_loss = np.append(plot_loss, [ma_loss])\n",
    "            \n",
    "            if num_iter % 100 == 0:\n",
    "                plt.plot(plot_iter, plot_loss)\n",
    "                display.clear_output(wait=True)\n",
    "                plt.show()\n",
    "                \n",
    "                sample_text = self.sample_output(self.char_to_idx[self.data[start_pos]], \n",
    "                                                 200)\n",
    "                print(sample_text)\n",
    "\n",
    "            num_iter += 1\n",
    "            \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD0CAYAAACPUQ0CAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hUVf7H8fdkUiGNlgahwxGkCtJDkSbg2nZdO+quyKprb2t3Ffuu7g/rLqzryoINbMsKKk0p0ns7EIq0JCS0hJI+vz9miKFIAiS5ZObzeh6fZ+6Zc8N3rsknJ+fee67L4/EgIiL+I8jpAkREpGIp2EVE/IyCXUTEzyjYRUT8jIJdRMTPBDtdgDEmDLgQSAOKHC5HRKS6cAOJwCJrbV7pNxwPdryhPtvpIkREqqkUYE7phnMh2NMAxo8fT0JCgtO1iIhUC+np6Vx//fXgy9DSzoVgLwJISEigQYMGTtciIlLdnDCFrZOnIiJ+RsEuIuJnFOwiIn5GwS4i4mcU7CIifqbaB7uWHRYROVa1DvZPF2/n0jfnsi4t2+lSRETOGdU62JvFRWIzchg2ejZjZ2/W6F1EhGoe7Bc0rMWMB/owoFU8o/63jhEfLCYjO9fpskREHFWtgx2gQa0avHtDJ+7u34Jp63Yz+G8/sCEjx+myREQcU+2DHSAoyMX9A1vy+R09CHEHccf4peQVaqFIEQlMfhHsR3VsWItXftOO1N0HeXfWZqfLERFxhF8FO0A/E8fQtgn8/YdNZObklb2DiIif8btgB3hwkCGvsJh3v9/kdCkiIlXOL4O9ab1ILmmXyCeLt5NboLl2EQksfhnsAFdfmExObiHT1mU4XYqISJXy22Dv0rg2NUPdLNyy1+lSRESqlN8Ge7A7iFaJ0dh0XdMuIoGlXI/GM8Z0BV621vY1xsQBY4BaeJ+SPdxau8kYMwIYCRQCo6y1k40xdYEJQASwC7jFWnu4Mj7IySTFRrB8+/6q+udERM4JZY7YjTEPA2OBcF/TK8B4a21v4AngPGNMAnA30BMYDLxojAkDngImWGtTgGV4g7/KJMVGkH4gl+JirSEjIoGjPFMxm4ArS233BBoYY6YB1wOzgC7AXGttnrX2AJAKtAN6AVN9+00BBlRQ3eWSFBtOflExWYd0PbuIBI4yg91aOwkoKNXUGNhnrR0AbAMeAaKBA6X65AAxx7UfbasySTERAKTt18JgIhI4zuTk6R7gK9/r/wKdgWwgqlSfKGD/ce1H26pMYqx39mjX/iNV+c+KiDjqTIJ9DjDU97o3sAZYCKQYY8KNMTFAK2A1MLdU3yHA7LMr9/TUj/WO2Hcd0IhdRALHmQT7A8BwY8w84GLgBWttOjAab3DPAB631uYCo4BrjDFzge7AmxVTdvnERIQQEeLWiF1EAkq5Lne01m4Fuvle/wQMPEmfMXgvgyzdloE3/B3hcrlIig0n7YCCXUQCh9/eoHRUUmwEO3XyVEQCiP8He0wEaZqKEZEA4vfBnhgbTubBPPILi50uRUSkSvh9sCfFRODxoIdci0jA8PtgT4jxXsuermAXkQDh98GeGKOblEQksPh/sPtuUkrXTUoiEiD8Ptgjw4KJCgsmTcEuIgHC74MdvPPsuklJRAJFwAR7eraW7hWRwBAQwR4XFc5uXRUjIgEiIII9ISaM3Tl5epKSiASEgAj2+Ohwioo97DmU73QpIiKVLiCCPS7Kd5OSrowRkQAQEMEeHx0GwK/enONwJSIilS8ggr1+rQinSxARqTIBEexxUeEMah0PwJH8IoerERGpXAER7AC/ap8EwJasQw5XIiJSuQIm2JvWqwnA5qyDDlciIlK5AibYm9T1BvtPew47XImISOUKmGCvERpM3cgwtinYRcTPBUywAyTXjmD7PgW7iPi3gAr2hrVrsG2vgl1E/FvABXvagVwKivRgaxHxXwEX7EXFHnbs09rsIuK/AirYm8VFArBpty55FBH/FVjBXs8b7KmZCnYR8V8BFewxESHUiwojVSN2EfFjARXsAC3iItmYkeN0GSIilSbggr1VYjTr03Mo1JUxIuKnAjLY8wqL+UnXs4uInwouTydjTFfgZWttX2NMR2AysNH39jvW2o+NMU8Dw4BC4F5r7UJjTHPgfcADrAbutNY6OlRuUrcGAFuzDpWcTBUR8SdljtiNMQ8DY4FwX1Mn4DVrbV/ffx8bYy4A+gBdgWuAt3x9XwOesNamAC7gsor+AKercR3vYmBvzkx1uBIRkcpRnhH7JuBKYJxvuxNgjDGX4R213wv0Ar611nqAbcaYYGNMPV/f7337TQEGAZ9XYP2nrU5kGH1a1mPR1r0UFXtwB7mcLEdEpMKVOWK31k4CCko1LQQestb2BjYDTwPRwIFSfXKAGMDlC/vSbY67rEMSh/OLdNmjiPilMzl5+rm1dsnR10BHIBuIKtUnCtgPFJ+kzXHtk2MBWLHjnChHRKRCnUmwf2OM6eJ73R9YAswFBhtjgowxDYEga20WsMwY09fXdwgw+2wLrghN6tQkKiyYFdsV7CLif8p1VcxxbgfeMMYUAOnAbdbabGPMbOBHvL8s7vT1fQAYY4wJBdYBEyug5rMWFOSiXXKMRuwi4pfKFezW2q1AN9/rpUDPk/R5BnjmuLYNeK+WOee0bxDLP37YTG5BEeEhbqfLERGpMAF3g9JR7ZNjKSz2sDYt2+lSREQqVOAGewPfCVTNs4uInwnYYE+ICSc+OkzBLiJ+J2CDHbyj9pU7DpTdUUSkGgnsYE+OZXPWIQ4cLii7s4hINRHQwd7Bd6PSyp2ajhER/xHQwd62gXeFg+XbFOwi4j8COtijw0NoHhfJMp1AFRE/EtDBDnBBw1hmrN/NhAXbnC5FRKRCBHywd25UG4DHPl+lZ6GKiF8I+GD/VfskhrVLBGDyyjSHqxEROXsBH+wRoW7euu4CujapzZTVCnYRqf4CPtiPGto2kQ0ZB/XwDRGp9hTsPgNbxwMwfV0Gz01ey7DRs0k7cMThqkRETp+C3ScpNoLWidF8sng7/5yzhTW7shn3409OlyUictoU7KUMaBXHpsxDJdszbaaD1YiInBkFeymXd6xPiNtF/dgIHhpsWJeWzfa9h50uS0TktCjYS2laL5K1z17MnEf6Mcg3557yykxyC4ocrkxEpPwU7McJcQfhcrloER/F1Z2TAZi4ZAcFRcVk5uTx8aJt3DB2AVuyDpXxlUREnHEmD7MOGC/9ui2bMg/yxBereeKL1ce897v3F/H5HT2IrRHqUHUiIienEfspuFwu/vrb9iXbbepHEx8dxqu/aceWrEN0ePY7bv7XQkZP34jH43GwUhGRn2nEXoZGdWryxZ09SYgOJyEmHI/Hg8vlIrewmCe/WM0sm8ksm4k7yMWd/Zo7Xa6IiEbs5dEhOZaEmHDAO4oHuLFbI9Y/dzFPDGtF/dgIxs7eTGFRsZNliogACvazEh7i5taUpjw2tBX7Dhew5Kd9TpckIqJgrwh9TD1C3C6mr9/tdCkiIgr2ihAZFky3pnWYti7D6VJERBTsFWVg63g2Zx5ifXq206WISIBTsFeQS9ol4Q5y8dXyXU6XIiIBTsFeQWrXDKVTo1paOExEHKdgr0D9z4tjXVo2U/UkJhFxkIK9At3UozG1aoTw6eIdTpciIgGsXHeeGmO6Ai9ba/uWarsOuMta2923PQIYCRQCo6y1k40xdYEJQASwC7jFWuu36+CGh7i5rEN9Ply4jcP5hdQI1Y29IlL1yhyxG2MeBsYC4aXaOgK/B1y+7QTgbqAnMBh40RgTBjwFTLDWpgDL8Aa/X+vfKo68wmIWbN7rdCkiEqDKMxWzCbjy6IYxpg7wAnBvqT5dgLnW2jxr7QEgFWgH9AKm+vpMAQZURNHnsgsb1yY8JIiPFm2jQEsMiIgDygx2a+0koADAGOMG/gncD+SU6hYNHCi1nQPEHNd+tM2vhYe4MfFRfLMmg6e/WuN0OSISgE735GknoAXwDvAR0NoY8zcgG4gq1S8K2H9c+9E2v3dh49oATFiwjfQDuQ5XIyKB5rSC3Vq70Fp7vu8k6jXAWmvtvcBCIMUYE26MiQFaAauBucBQ3+5DgNkVVvk57KGLDQ8NNoS4Xfxt2obT3n9eahY3/nMBO/b57XlmEalEFXK5o7U2HRiNN7hnAI9ba3OBUcA1xpi5QHfgzYr49851YcFu7uzXnEvaJfH1qjTyCst+Zur8zXt44JMVHM4v5G/TNjJ7Yxb/nLOlCqoVEX9TruvxrLVbgW6narPWjgHGHNcnA7j4bIusri7rkMTny3byw4YsBvoejn0y363N4NHPVpJ1MJ/C4mIWbvVeUTN+wTbuG9iS6PCQqipZRPyALrSuRD2b1yUmIoSvV6UxsHU8u/Yf4dPFO7isQxLLtu9j574jLN9+gGnrMghxex/g8aVvrZm/XtWeBz5dwTuzNrFqxwE8eLi2S0MGtIonPMTt5McSkXOcgr0ShbiDGHx+PF+vSuf7DZk8PHEFGdl5vH7cvHunRrX46LZuzEnN4pZ/LeLKjvVJaVEXgHdmbSrpNzd1D2HBQQw6P4Ebujaka9M6Vfp5RKR6ULBXsqFtE/lk8Q5uem8hUeHBnJcQRXx0OCP7NKVJ3ZpkHymkQa0IQtxB9DNxrH/OO3MVHuKmdWI0a9Oyefhiw5bMQ9SvFcHfpm3kvyt2MWVVGjMe6EvDOjUc/oQicq5RsFeyns3rlrz+5t7eJMVGHPN+4nFX9peeZhn3+y7M27SHX7VPAiC3oIhv1mQwrG0Cf/l2A9+uTefWlKaVV7yIVEsK9koW4g5i60vDzmjfOpFhJaEO3tCfck8KAB8u3M6o/63j2zUZ/KZTA8JCghjWNpGZNpPNmQcZ1i6RHfuOYOKjqFUztEI+i4hUDwr2aureAS14aOJKFm7dW3IVzT0fLS95/8Up6wHvOvFLnhiAy+VypE4RqXpatreauqpzMhufH8LHt3UjPjqMS0uN7K/v2rDk9d5D+ezOyXOiRBFxiEbs1ViIO4iuTeuw4DHv2mpdmtRmXVo2fxpyHpHhwbSrH8udE5YyZVUaN/ds4nC1IlJVFOx+5IZujUpePzqkFfmFxVzQMJaXp1oa1KqBSYgiubauohHxdwp2PxYaHMTzV7TlsrfmcusHi3EHufhkZHfqRYbx5JeriY4IoU7NUOpFhXFh49qk7j7Ibzs34FB+EfM376FX87rUDNO3iEh1o59aP9cqMZpZD/Zl4Za93Pvxcn79zrxT9n9jxkbSfCtSjuzTlEeHtKqKMkWkAunkaQBIio3g8o71uei8uJK2yXf1Yso9Kcz900Vc2bE+8dFhXNslmWb1Ikv6/OfHn9h7KN+JkkXkLGjEHkD+elV7fv3uPO66qDlt6v98Z9RrV3c4pl/6gVy2ZB3i2jHz+XDhNu7s17yqSxWRs6BgDyC1aoYy44G+ZfZLiAknISacHs3q8K+5W7m2S0Nq6yYnkWpDUzHyix4f1ooDR/J56NMV5Bfq+a0i1YWCXX7R+UkxPDz4PKav3831Y+czLzWLjRk5Ze8oIo7SVIyc0q0pTdi5/wjvz9vKdWMX4A5yse7ZizmUV6g1aETOURqxyym5XC7uG9CS9g28J1uLij20fGIKHZ/7jmXb9jlcnYicjIJdyhRTI4Qv7uzJ2mcH88dSV8jMTc1ysCoR+SUKdikXl8tFjdBgHhjUkjq+KZil2/Y7XJWInIyCXU6Ly+ViyZMDubpzMgs27+H+j5dz/yfLKSr2OF2aiPjo5KmckRu7N+Ljxdv5bNlOAIJcLp68pDUxESEOVyYiGrHLGWlTP4YRKU1wB7m4tH0SE5fs4JI3ZuPxaOQu4jSN2OWMPTqkFQ8ONhQVe1i2fR/b9x5hza7sY5YrEJGqpxG7nLGgIBdhwW5qhAYz6fYeAFzyxhwKi3SXqoiTFOxSIeKiwkteb8465GAlIqJglwoz7f4+AIyevlFz7SIOUrBLhWkeF8mVHeszeWUa8zfvdbockYClYJcK9dSvWlMz1M3bs1KdLkUkYCnYpULF1gjl0g5J/LhpD0fyi5wuRyQgKdilwg06P4HCYg//mrfF6VJEAlK5rmM3xnQFXrbW9jXGtAb+AbiAjcCt1tpCY8wIYCRQCIyy1k42xtQFJgARwC7gFmvt4cr4IHLu6GfiuKBhLK9MtXRMrkX3ZnWcLkkkoJQ5YjfGPAyMBY5ez/YC8Ji1tqdv+1fGmATgbqAnMBh40RgTBjwFTLDWpgDL8Aa/BICHBp8HwLVj5pN+INfhakQCS3mmYjYBV5ba/rW19gdjTCiQABwAugBzrbV51toDQCrQDugFTPXtNwUYUGGVyzmte7M6vHFtRwCmrE5zuBqRwFJmsFtrJwEFpbaLjDGNgDVAXWAFEI034I/KAWKOaz/aJgHiknaJNK5Tgz//dy0fL9pGboFOpopUhTM6eWqt/cla2wJ4F3gNyAaiSnWJAvYf1360TQKEy+Xi2cvaAPDIpFWc9+RU3pqZyrY9h/l6VZpuYhKpJKcd7MaYr4wxLXybOUAxsBBIMcaEG2NigFbAamAuMNTXdwgw++xLluqkd8t6/PBQP7o0qU1ocBCvfmPp/epM7hi/lElLdzpdnohfOpMR+0vA+8aYmcBwvCdS04HReIN7BvC4tTYXGAVcY4yZC3QH3qyYsqU6aVinBp+M7M6KpwbRq3ndkvanvlztYFUi/svl9J/DxpjGwJbp06fToEEDR2uRyncor5A5qVl8uyaDSUt3sPCx/sRFh5e9o4gcY8eOHfTv3x+gibV2a+n3dIOSVKmaYcEMPj+BkX2aEuoO4rHPNWoXqWgKdnFEy/gohndvxA8bMskv1PrtIhVJwS6O6dSoFvlFxfywIdPpUkT8ih6NJ465qFUcTevW5Omv1pAQE06Qy0XrpGinyxKp9jRiF8eEBbt5/oq27M7J5ZI35jB09Gx252j5AZGzpWAXR3VvVoePbutesn3pG3Ox6TkOViRS/SnYxXGdGtVi5TODmDCiK4fyC7n0zTls36tFQEXOlIJdzgnR4SH0aFaXpy5pTV5hMdf8Yz5vz0rVwzpEzoCCXc4pV3VOZuIfurNz/xFemWoZ/t4Cioo95BUW8cAnKxg2ejZzNmY5XabIOU1Xxcg5p3Pj2gxoFc+0dRks2rqP4e8tYP/hAtbsygbg7o+WMbJ3Uy7rUJ+EGN21KnI8jdjlnDT2ps5sfWkYf+zXnLmpe1izK5vh3Rvx8q/bsvdQPi9OWU+3F6ezeueBsr+YSIDRiF3OaQ8ONlzesT7REcHERYWTW1DEd2t3M21dBgCTV6bRpv7pLfPv8XgoKPJQUFRMzTD9CIj/0Xe1nPOax0WWvA4PcTP2ps4A9H11Ju9+v4noiGBu7NaIqPCQU36dqavTmbYug32H8pm+fjehwUF8OKIrnRrVrtT6RaqapmKk2rq2S0MAXplq+eOEZSftU1BUzP9N28gVb8/lD/9ZwsQlO5i+fjcA+YXF3DF+qZ7sJH5HwS7V1sg+zdjy4lB+36sJ32/IPObad4/HwzNfrcE8MYXXp21g2Tbvw7siw4J5/oo2bH1pGO9cfwEZ2XkM/+dCios9eqKT+A0Fu1RrLpeLS9snAXDNP+azMcN71+qCLXt5f95Win1ZfXOPxsx4oA+r/zyY67s2AuDiNglc3iGJhVv30vSxr+n50gzyCjV6l+pPc+xS7bVPjuWJYa0Y9b91DHz9B9onxxId7v3Wvrt/Cwa1jj/pCVaXy8Wzl7fBZhxkXVo2uw7ksnzbfro2rVPVH0GkQmnELn7h972a8NZ1FwCwYvt+Zm/M4vquDbl/YMtTXjUTHR7ClHtSWPH0IKLDg3lk0kq+XL5Td7xKtaZgF7/gcrkY1i6RVc8MItF309Kd/ZqXe/+YiBDeuv4Cduw7wj0fLeeO8UvweDw6sSrVkqZixK9EhYcw8fYeHDhcQFJsxGntm9KiHjMf7MtfvrV8uXwXTR79GoAHB7Xkjxe1OO1aUncfJDIsWHfHSpXTiF38Tv3YiDN+YEdy7Ro8e2kbRvZuWtL2r7lbT/uKmVGT1zLgte/p8dJ0Cov06D+pWgp2kePE1Ajh0aGt2PrSMJ67vA17DuXz+rSNHM4vLNf+Bw4XMHbOFgCKPbAuTevLS9VSsIucwpUd69OlSW1GT99Ir5dnnvQhIMXFHjJz8liz6wA5uQX0fHkGAO/e0Al3kIvJK3dVddkS4DTHLnIKNcOC+WRkdxZt3cvv/rWI17/bwDs3XIDL5SrpM/I/S/hubcYJ+w5oFcfg8+MZv2AbV1+YTNN6kSf0EakMGrGLlMOFjWszvEcjpq5J54kvVpe0Z+bklYT6kDYJ3NqrCa/9tj121MUEu4N4aPB5BLtd/Pbv81m1o2JXovR4PLz+3QZu/88S3Vglx9CIXaScHhxk2HMwn48Wbef+gS3ZdzifAa/9AMC7N1zAxW0ST9inSd2ajL6mI7eNW8yf/7uGqy9MZlDrBGJq/PKCZQVFxeQXlr3y5L/nbeX/pm8E4PwfNp/RlTvinzRiFyknl8vFrSlNKPZ4uP+TFSWhDtCnZdwv7te7ZT1u6tGYxT/t46GJK2n/7Le/+EzXQ3mFXPOP+fR6eQa7c3LZuf8Ii7buPaHfnoN5PPPftQBEhwfz1sxNHMwr38ld8X8KdpHT0DwuiuHdGvH9hsyStlkP9iUi1H3K/a7unHzM9rDRs9mSdeiEfp8t28mSn/ax73ABXZ6fTs+XZnDVuz/S/6+zSh4qUlTsKVnN8urOybxzQyeOFBTxvc084etJYNJUjMhpeubS80muXYOM7FweH9a6XPs0rRfJl3f25HB+EV8s28nHi7fzzFdrCHG7SIqNoFfzumQdzOfNGRtpUCuCbk3rMGdjFunZuQBs33eEK96eyzUXNuSzpTs4lF9Ecu0Inr+iDR68Uz5//dZycZsE3EGuUxcjfk/BLnKavFMyTcvueJz2ybEAdG9Wh4Z1avDqN7bkvQ9+/Knk9Xs3d+ai8+IByCssItQdxJasQ/zhP0sYN9/b766LmvPAIFOyz+19mvHwpJVsyTpI87ioM/pc4j8U7CIOuLlHY96amcrh/CLaJ8eyLi2b+wa0pE39aFJa1CvpFxbsneJpWi+SSbf3wKbncH5SzAlTP0d/afxxwjLOT4qhRXwk+w8XUDPUzV39dVI10CjYRRxQMyyY2Q/3Y/u+I3TwhXJZosJD6Nz45I/xaxEXSWJMOOvTc1h/3E1UI/s047nJaxm/4Cc6NqzFw4O9I30tT+y/yhXsxpiuwMvW2r7GmA7AG0ARkAcMt9ZmGGNGACOBQmCUtXayMaYuMAGIAHYBt1hrT345gEiAqRMZRp3IsAr5WkFBLj4c0Y3Plu6g33lxPPnlajZmHCSvsJiWT0wp6bfkp31c/Y/5AKQ+P4Rgt66f8Edl/l81xjwMjAWOLlH3f8Bd1tq+wGfAI8aYBOBuoCcwGHjRGBMGPAVMsNamAMvwBr+IVILGdWty/yBDx4a1mHxXCmv+PJiLz08oeb9t/Rg+v6MHLeO9d8DeNm7JL152KdVbeUbsm4ArgXG+7WustWml9s8FugBzrbV5QJ4xJhVoB/QCXvD1neJ7/XoF1S4ipxDsDuLdGzud0D7lnt489eVqxi/YxsIte+nWtDZXdU5mcKlfAlK9lTlit9ZOAgpKbacBGGN6AH/EG9TRQOn7pXOAmOPaj7aJiIPcQS5GXd6GscM7UycylGnrdjNynJYl8CdnNMFmjLkaeBcYZq3NBLKB0tdYRQH7j2s/2iYiDnO5XAxoHc+k23vQuE4NAOamZpVr3xXb9/Pl8p2nvUa9VJ3TDnZjzA14R+p9rbWbfc0LgRRjTLgxJgZoBawG5gJDfX2GALPPvmQRqSh1I8P4+p4UkmtHMHLcEv7yjeW5yWtZvfMAHo+H5yavpfuL05nmW+jsy+U7ueytudzz0XL6/mUWu/YfcfgTyMm4yvNb1xjTGPgI78nRTGAbP4++v7fWPu27KuY2vL8sXrDWTjLGxAP/xjtazwKus9YeOsnX3jJ9+nQaNGhQIR9KRE7P2l3ZDB197LhrZJ+m/P37zSf0TYoJJye3kJy8Qur7Hj/4wpVt6dOy3gl9pfLs2LGD/v37AzSx1m4t/V65gr0yKdhFzg0Z2blMXLKDOjVD+dNnqwCICHEzYURXrnh7HgDtGsTw2e09KCz28OzktUxYsK1k/7XPDqZGaDDjF/xExoFc7hvY8ph166VinSrYdYOSiAAQHx3Onf2aA95nv46ZvZnLOiTRsWEtPvhdFxZt3ct9A1oSFOQi2A0vXNGWuy9qQbcXpwMwctwS/n5jJx7/3Lte/epd2Yy+tiORZSw/LBVPI3YROSu79h/hkUkrmb3xxJOvzeMiefU37YitEUqTujUdqM5/nWrErtvOROSsJMVG8Ma1Hanru4v26s7JfHdfb/7Yrzl7D+Vzxdvz6PeXWZz35BTG/bjV0VoDhf5GEpGzFlsjlNkP9+NwfmHJMgkPDjbE1ghh1P/WAZBbUMyTX67hV+2TiK0R6mS5fk/BLiIVIiLUfcKqk7f0bEKIO4gWcZG88o1l+fb99Hl1Fs9c2pruTeuSEBN+TH+Px8NTX65h76F8UlrUpY+pR2JMRFV+DL+gYBeRSuMOcnFTj8YAfN6sDjf+cyFzUrO47+MVAEwY0ZUezeqW9J+1IbNkzfn/rfKuXDJmeGcGto6v2sKrOc2xi0iVcLlcjL2pM9d1bciQNt51aa4bs4BPF2/ncH4hGzNyeGTiSqLCghnSJoHLOiQBMOKDxdzz0TLyC4udLL9a0VUxIuKIr1elccf4pce0hYcEMfEPPWhT37us1M79R+j50gwAPvhdF3rrJqgSuipGRM45Q9smMu3+3sTWCOGChrHc078Fk+9KKQl1gPqxEbxwRVsAHv1sFW/NTCX9QK5TJVcbmmMXEcc0j4ti/qP9CQsO+sW7VK/r2pBd+4/w5sxUXv3GsmzbPsbedGEVV1q9KNhFxFHhIe4y+zw42DCsXSJ//34TXyzfxa3/XoTL5V1+OD46vMz9A42mYkSkWmiVGM01XRoCMG3dbr5bm0HXF6bz46Y9Z/21PR4PY37YzOKtexscav0AAAawSURBVM/6a50LNGIXkWqjW9M6zPvTRURHhPDfFbt45qs1XDtmPq/8ph1D2yae0bo009dlcPt/lpJf9PNVNz8+ehEJ0eHVdhEzjdhFpFpJio0gMiyYa7s05L2bvXPtD09cyVNfrD7tr/XFsp2M+GAx+UXFmPifnxXU/cUZvD5tY4XVXNUU7CJSbfVsXpcVTw8iLiqMz5btZNKSHSftV1hUzDNfrSF1d84x7f+at5WW8VEsenwAU+9NYcOoIYy6vA0Ab89MZe+h/HLX8tnSHfzHd3MVwObMg6zc4cxD4zQVIyLVWkxECO/f0oWho2fzwKcr+HTJdpJr1eD7DZk8MKglAOPm/8TqndnY9BxGXdGGZvUiWbF9Pyu27+e23k2pF+Vd3yY02MUN3RrRrWltBr7+A6P+t5bXftuhzBo2Zx7k/k+8d9OOmb2ZoW0TeWfWJgCm3JNCq8ToSvr0J6cblETEL+zOyWXs7C18vmwnmTl5p+wbFxXGbl+fuX+6qORJUKU9+cVqxs3/ibsuas79v/DQkMKiYt6YkcqHC7eRdTAPd5CLgqJjM7V+bASzHupLiLtiJ0h0g5KI+L24qHAeG9qKj27rVrIcwSXtEnlwUEu+uLMnY4d3Lul7NNQHnx9/0lAHuHdACxrUiuCNGal0f3EGRcU/B/aeg3nc9eEy7v9kBf83fSO7c/J4bGgrNj4/lA9HdCMpJpz3bu7M3f1bsHP/EVo8PqVKH/6tEbuI+KXiYg9BQSe/qiW3oIjU3QePucv1ZIqKPXR7cTqZOXl0aVyb9393IevSchg5bglZB3/+q+D1q9tzWfv6J/x7uQVFnPfk1JLt+wa05DedG/ziL5PToUfjiUjA+aVQB+9NUWWFOnhXp/zxTxdxy/uLmL0xixe/Xs/CLXvJOpjH73s1YVPmQW7s1oj+rU6++mR4iJulTw7k1n8vYum2/bw+bQNTVqfRrWkdbHoOK3fsZ+q9vUmuXeOMP+fJKNhFRE4h2B3EP27szEV/nVWypPCIlCY8Pqx1ufavXTOUz+7oSeruHF77bgNfr0pnfXoO0eHBXNU5maQKGL0fT3PsIiJliAh1c0e/5oS6g3jpyrblDvXSmsdFcXf/FoS6g7iyY31WPjOYZy49H/cp/rI4Uxqxi4iUww1dG3JD14ZndTfqeQnRLHisP5HhlRu9CnYRkXKoqOUFatWs/Oe9aipGRMTPKNhFRPyMgl1ExM8o2EVE/IyCXUTEzyjYRUT8zLlwuaMbID093ek6RESqjVKZecJDY8+FYE8EuP76652uQ0SkOkoENpVuOBeCfRGQAqQBRQ7XIiJSXbjxhvqi499wfNleERGpWDp5KiLiZ86FqZgzYowJAt4G2gN5wK3W2lRnq6p8xpgQ4D2gMRAGjALWAu8DHmA1cKe1ttgY8zQwDCgE7rXWLnSi5spmjIkDlgAD8X7W9wncY/EocCkQivfn43sC9Hj4flb+jfdnpQgYQYB8f1TnEfvlQLi1tjvwJ+CvDtdTVW4A9lhrU4CLgTeB14AnfG0u4DJjzAVAH6ArcA3wlkP1VirfD+/fgSO+pkA+Fn2BHkBPvJ83mQA+HsBQINha2wN4FnieADke1TnYewFTAay184HOp+7uNz4FnvS9duEdYXTCOzIDmAIMwHt8vrXWeqy124BgY0y9qi62CvwFeBfY5dsO5GMxGFgFfA78F5hMYB+PDXg/WxAQDRQQIMejOgd7NHCg1HaRMabaTi2Vl7X2oLU2xxgTBUwEngBc1tqjZ8FzgBhOPD5H2/2GMeZmINNa+02p5oA8Fj518Q5wrgL+AIwHggL4eBzEOw2zHhgDjCZAvj+qc7BnA1GltoOstYVOFVOVjDHJwExgnLV2AlBc6u0oYD8nHp+j7f7kd8BAY8wsoAPwARBX6v1AOhYAe4BvrLX51loL5HJsQAXa8bgP7/Foifdc3L/xnns4ym+PR3UO9rl459AwxnTD+yeo3zPGxAPfAo9Ya9/zNS/zza8CDAFm4z0+g40xQcaYhnh/8WVVecGVyFrb21rbx1rbF1gODAemBOKx8JkDXGyMcRljkoCawPQAPh77+HkkvhcIIUB+Vqrz1MXneEdr8/DONd/icD1V5TGgFvCkMeboXPs9wGhjTCiwDphorS0yxswGfsT7C/xOR6qteg8AYwLxWFhrJxtjegML+flzbiFAjwfwOvCe77OG4v3ZWUwAHA/doCQi4meq81SMiIichIJdRMTPKNhFRPyMgl1ExM8o2EVE/IyCXUTEzyjYRUT8jIJdRMTP/D9wSrOKuJY1uQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r\n",
      "Bhy wiund hory cov Cnot un dul, koras daunt irlt -he,g tom tas me wootg; nod ynXohi nhe soind woth moy has wirsA:no nould poie H.ls fonvas:\n",
      "ysd mI mut ithin eere ? youl hoetUothe bernd le cort co lo\n"
     ]
    }
   ],
   "source": [
    "layers = [RNNLayer(hidden_size=256, output_size=62)]\n",
    "mod = RNNModel(layers=layers,\n",
    "               vocab_size=62, sequence_length=10,\n",
    "               loss=SoftmaxCrossEntropy())\n",
    "optim = SGD(lr=0.001, gradient_clipping=True)\n",
    "trainer = RNNTrainer('input.txt', mod, optim)\n",
    "trainer.train(1000, sample_every=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With RNN cells, this gets stuck in a local max. Let's try `LSTM`s."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSTMs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `LSTMNode`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTMNode:\n",
    "\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        param hidden_size: int - the number of \"hidden neurons\" in the LSTM_Layer of which this node is a part.\n",
    "        param vocab_size: int - the number of characters in the vocabulary of which we are predicting the next\n",
    "        character.\n",
    "        '''\n",
    "        pass\n",
    "        \n",
    "    def forward(self, \n",
    "                X_in: ndarray, \n",
    "                H_in: ndarray, \n",
    "                C_in: ndarray, \n",
    "                params_dict: Dict[str, Dict[str, ndarray]]):\n",
    "        '''\n",
    "        param X_in: numpy array of shape (batch_size, vocab_size)\n",
    "        param H_in: numpy array of shape (batch_size, hidden_size)\n",
    "        param C_in: numpy array of shape (batch_size, hidden_size)\n",
    "        return self.X_out: numpy array of shape (batch_size, output_size)\n",
    "        return self.H: numpy array of shape (batch_size, hidden_size)\n",
    "        return self.C: numpy array of shape (batch_size, hidden_size)\n",
    "        '''\n",
    "        self.X_in = X_in\n",
    "        self.C_in = C_in\n",
    "\n",
    "        self.Z = np.column_stack((X_in, H_in))\n",
    "        \n",
    "        self.f_int = np.dot(self.Z, params_dict['W_f']['value']) + params_dict['B_f']['value']\n",
    "        self.f = sigmoid(self.f_int)\n",
    "        \n",
    "        self.i_int = np.dot(self.Z, params_dict['W_i']['value']) + params_dict['B_i']['value']\n",
    "        self.i = sigmoid(self.i_int)\n",
    "        self.C_bar_int = np.dot(self.Z, params_dict['W_c']['value']) + params_dict['B_c']['value']\n",
    "        self.C_bar = tanh(self.C_bar_int)\n",
    "\n",
    "        self.C_out = self.f * C_in + self.i * self.C_bar\n",
    "        self.o_int = np.dot(self.Z, params_dict['W_o']['value']) + params_dict['B_o']['value']\n",
    "        self.o = sigmoid(self.o_int)\n",
    "        self.H_out = self.o * tanh(self.C_out)\n",
    "\n",
    "        self.X_out = np.dot(self.H_out, params_dict['W_v']['value']) + params_dict['B_v']['value']\n",
    "        \n",
    "        return self.X_out, self.H_out, self.C_out \n",
    "\n",
    "\n",
    "    def backward(self, \n",
    "                 X_out_grad: ndarray, \n",
    "                 H_out_grad: ndarray, \n",
    "                 C_out_grad: ndarray, \n",
    "                 params_dict: Dict[str, Dict[str, ndarray]]):\n",
    "        '''\n",
    "        param loss_grad: numpy array of shape (1, vocab_size)\n",
    "        param dh_next: numpy array of shape (1, hidden_size)\n",
    "        param dC_next: numpy array of shape (1, hidden_size)\n",
    "        param LSTM_Params: LSTM_Params object\n",
    "        return self.dx_prev: numpy array of shape (1, vocab_size)\n",
    "        return self.dH_prev: numpy array of shape (1, hidden_size)\n",
    "        return self.dC_prev: numpy array of shape (1, hidden_size)\n",
    "        '''\n",
    "        \n",
    "        assert_same_shape(X_out_grad, self.X_out)\n",
    "        assert_same_shape(H_out_grad, self.H_out)\n",
    "        assert_same_shape(C_out_grad, self.C_out)\n",
    "\n",
    "        params_dict['W_v']['deriv'] += np.dot(self.H_out.T, X_out_grad)\n",
    "        params_dict['B_v']['deriv'] += X_out_grad.sum(axis=0)\n",
    "\n",
    "        dh_out = np.dot(X_out_grad, params_dict['W_v']['value'].T)        \n",
    "        dh_out += H_out_grad\n",
    "                         \n",
    "        do = dh_out * tanh(self.C_out)\n",
    "        do_int = dsigmoid(self.o_int) * do\n",
    "        params_dict['W_o']['deriv'] += np.dot(self.Z.T, do_int)\n",
    "        params_dict['B_o']['deriv'] += do_int.sum(axis=0)\n",
    "\n",
    "        dC_out = dh_out * self.o * dtanh(self.C_out)\n",
    "        dC_out += C_out_grad\n",
    "        dC_bar = dC_out * self.i\n",
    "        dC_bar_int = dtanh(self.C_bar_int) * dC_bar\n",
    "        params_dict['W_c']['deriv'] += np.dot(self.Z.T, dC_bar_int)\n",
    "        params_dict['B_c']['deriv'] += dC_bar_int.sum(axis=0)\n",
    "\n",
    "        di = dC_out * self.C_bar\n",
    "        di_int = dsigmoid(self.i_int) * di\n",
    "        params_dict['W_i']['deriv'] += np.dot(self.Z.T, di_int)\n",
    "        params_dict['B_i']['deriv'] += di_int.sum(axis=0)\n",
    "\n",
    "        df = dC_out * self.C_in\n",
    "        df_int = dsigmoid(self.f_int) * df\n",
    "        params_dict['W_f']['deriv'] += np.dot(self.Z.T, df_int)\n",
    "        params_dict['B_f']['deriv'] += df_int.sum(axis=0)\n",
    "\n",
    "        dz = (np.dot(df_int, params_dict['W_f']['value'].T)\n",
    "             + np.dot(di_int, params_dict['W_i']['value'].T)\n",
    "             + np.dot(dC_bar_int, params_dict['W_c']['value'].T)\n",
    "             + np.dot(do_int, params_dict['W_o']['value'].T))\n",
    "    \n",
    "        dx_prev = dz[:, :self.X_in.shape[1]]\n",
    "        dH_prev = dz[:, self.X_in.shape[1]:]\n",
    "        dC_prev = self.f * dC_out\n",
    "\n",
    "        return dx_prev, dH_prev, dC_prev"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `LSTMLayer`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTMLayer:\n",
    "\n",
    "    def __init__(self,\n",
    "                 hidden_size: int,\n",
    "                 output_size: int,\n",
    "                 weight_scale: float = 0.01):\n",
    "        '''\n",
    "        param sequence_length: int - length of sequence being passed through the network\n",
    "        param vocab_size: int - the number of characters in the vocabulary of which we are predicting the next\n",
    "        character.\n",
    "        param hidden_size: int - the number of \"hidden neurons\" in the LSTM_Layer of which this node is a part.\n",
    "        param learning_rate: float - the learning rate\n",
    "        '''\n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        self.weight_scale = weight_scale\n",
    "        self.start_H = np.zeros((1, hidden_size))\n",
    "        self.start_C = np.zeros((1, hidden_size))        \n",
    "        self.first = True\n",
    "\n",
    "        \n",
    "    def _init_params(self,\n",
    "                     input_: ndarray):\n",
    "        \n",
    "        self.vocab_size = input_.shape[2]\n",
    "\n",
    "        self.params = {}\n",
    "        self.params['W_f'] = {}\n",
    "        self.params['B_f'] = {}\n",
    "        self.params['W_i'] = {}\n",
    "        self.params['B_i'] = {}\n",
    "        self.params['W_c'] = {}\n",
    "        self.params['B_c'] = {}\n",
    "        self.params['W_o'] = {}\n",
    "        self.params['B_o'] = {}        \n",
    "        self.params['W_v'] = {}\n",
    "        self.params['B_v'] = {}\n",
    "        \n",
    "        self.params['W_f']['value'] = np.random.normal(loc=0.0,\n",
    "                                                       scale=self.weight_scale,\n",
    "                                                       size =(self.hidden_size + self.vocab_size, self.hidden_size))\n",
    "        self.params['B_f']['value'] = np.random.normal(loc=0.0,\n",
    "                                                       scale=self.weight_scale,\n",
    "                                                       size=(1, self.hidden_size))\n",
    "        self.params['W_i']['value'] = np.random.normal(loc=0.0,\n",
    "                                                       scale=self.weight_scale,\n",
    "                                                       size=(self.hidden_size + self.vocab_size, self.hidden_size))\n",
    "        self.params['B_i']['value'] = np.random.normal(loc=0.0,\n",
    "                                                      scale=self.weight_scale,\n",
    "                                                      size=(1, self.hidden_size))\n",
    "        self.params['W_c']['value'] = np.random.normal(loc=0.0,\n",
    "                                                      scale=self.weight_scale,\n",
    "                                                      size=(self.hidden_size + self.vocab_size, self.hidden_size))\n",
    "        self.params['B_c']['value'] = np.random.normal(loc=0.0,\n",
    "                                                      scale=self.weight_scale,\n",
    "                                                      size=(1, self.hidden_size))\n",
    "        self.params['W_o']['value'] = np.random.normal(loc=0.0,\n",
    "                                                      scale=self.weight_scale,\n",
    "                                                      size=(self.hidden_size + self.vocab_size, self.hidden_size))\n",
    "        self.params['B_o']['value'] = np.random.normal(loc=0.0,\n",
    "                                                      scale=self.weight_scale,\n",
    "                                                      size=(1, self.hidden_size))       \n",
    "        self.params['W_v']['value'] = np.random.normal(loc=0.0,\n",
    "                                                      scale=self.weight_scale,\n",
    "                                                      size=(self.hidden_size, self.output_size))\n",
    "        self.params['B_v']['value'] = np.random.normal(loc=0.0,\n",
    "                                                      scale=self.weight_scale,\n",
    "                                                      size=(1, self.output_size))\n",
    "        \n",
    "        for key in self.params.keys():\n",
    "            self.params[key]['deriv'] = np.zeros_like(self.params[key]['value'])\n",
    "        \n",
    "        self.cells = [LSTMNode() for x in range(input_.shape[1])]\n",
    "\n",
    "\n",
    "    def _clear_gradients(self):\n",
    "        for key in self.params.keys():\n",
    "            self.params[key]['deriv'] = np.zeros_like(self.params[key]['deriv'])\n",
    "                    \n",
    "        \n",
    "    def forward(self, x_seq_in: ndarray):\n",
    "        '''\n",
    "        param x_seq_in: numpy array of shape (batch_size, sequence_length, vocab_size)\n",
    "        return x_seq_out: numpy array of shape (batch_size, sequence_length, vocab_size)\n",
    "        '''\n",
    "        if self.first:\n",
    "            self._init_params(x_seq_in)\n",
    "            self.first=False\n",
    "        \n",
    "        batch_size = x_seq_in.shape[0]\n",
    "        \n",
    "        H_in = np.copy(self.start_H)\n",
    "        C_in = np.copy(self.start_C)\n",
    "        \n",
    "        H_in = np.repeat(H_in, batch_size, axis=0)\n",
    "        C_in = np.repeat(C_in, batch_size, axis=0)        \n",
    "\n",
    "        sequence_length = x_seq_in.shape[1]\n",
    "        \n",
    "        x_seq_out = np.zeros((batch_size, sequence_length, self.output_size))\n",
    "        \n",
    "        for t in range(sequence_length):\n",
    "\n",
    "            x_in = x_seq_in[:, t, :]\n",
    "            \n",
    "            y_out, H_in, C_in = self.cells[t].forward(x_in, H_in, C_in, self.params)\n",
    "      \n",
    "            x_seq_out[:, t, :] = y_out\n",
    "    \n",
    "        self.start_H = H_in.mean(axis=0, keepdims=True)\n",
    "        self.start_C = C_in.mean(axis=0, keepdims=True)        \n",
    "        \n",
    "        return x_seq_out\n",
    "\n",
    "\n",
    "    def backward(self, x_seq_out_grad: ndarray):\n",
    "        '''\n",
    "        param loss_grad: numpy array of shape (batch_size, sequence_length, vocab_size)\n",
    "        return loss_grad_out: numpy array of shape (batch_size, sequence_length, vocab_size)\n",
    "        '''\n",
    "        \n",
    "        batch_size = x_seq_out_grad.shape[0]\n",
    "        \n",
    "        h_in_grad = np.zeros((batch_size, self.hidden_size))\n",
    "        c_in_grad = np.zeros((batch_size, self.hidden_size))        \n",
    "        \n",
    "        num_chars = x_seq_out_grad.shape[1]\n",
    "        \n",
    "        x_seq_in_grad = np.zeros((batch_size, num_chars, self.vocab_size))\n",
    "        \n",
    "        for t in reversed(range(num_chars)):\n",
    "            \n",
    "            x_out_grad = x_seq_out_grad[:, t, :]\n",
    "\n",
    "            grad_out, h_in_grad, c_in_grad = \\\n",
    "                self.cells[t].backward(x_out_grad, h_in_grad, c_in_grad, self.params)\n",
    "        \n",
    "            x_seq_in_grad[:, t, :] = grad_out\n",
    "        \n",
    "        return x_seq_in_grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `LSTMModel`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTMModel(object):\n",
    "    '''\n",
    "    The Model class that takes in inputs and targets and actually trains the network and calculates the loss.\n",
    "    '''\n",
    "    def __init__(self, \n",
    "                 layers: List[LSTMLayer],\n",
    "                 sequence_length: int, \n",
    "                 vocab_size: int, \n",
    "                 hidden_size: int,\n",
    "                 loss: Loss):\n",
    "        '''\n",
    "        param num_layers: int - the number of layers in the network\n",
    "        param sequence_length: int - length of sequence being passed through the network\n",
    "        param vocab_size: int - the number of characters in the vocabulary of which we are predicting the next\n",
    "        character.\n",
    "        param hidden_size: int - the number of \"hidden neurons\" in the each layer of the network.\n",
    "        '''\n",
    "        self.layers = layers\n",
    "        self.vocab_size = vocab_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.sequence_length = sequence_length\n",
    "        self.loss = loss\n",
    "        for layer in self.layers:\n",
    "            setattr(layer, 'sequence_length', sequence_length)\n",
    "\n",
    "        \n",
    "    def forward(self, \n",
    "                x_batch: ndarray):\n",
    "        '''\n",
    "        param inputs: list of integers - a list of indices of characters being passed in as the \n",
    "        input sequence of the network.\n",
    "        returns x_batch_in: numpy array of shape (batch_size, sequence_length, vocab_size)\n",
    "        '''       \n",
    "        \n",
    "        for layer in self.layers:\n",
    "\n",
    "            x_batch = layer.forward(x_batch)\n",
    "                \n",
    "        return x_batch\n",
    "        \n",
    "    def backward(self, \n",
    "                 loss_grad: ndarray):\n",
    "        '''\n",
    "        param loss_grad: numpy array with shape (batch_size, sequence_length, vocab_size)\n",
    "        returns loss: float, representing mean squared error loss\n",
    "        '''\n",
    "\n",
    "        for layer in reversed(self.layers):\n",
    "\n",
    "            loss_grad = layer.backward(loss_grad)\n",
    "            \n",
    "        return loss_grad\n",
    "                \n",
    "    def single_step(self, \n",
    "                    x_batch: ndarray, \n",
    "                    y_batch: ndarray):\n",
    "        '''\n",
    "        The step that does it all:\n",
    "        1. Forward pass & softmax\n",
    "        2. Compute loss and loss gradient\n",
    "        3. Backward pass\n",
    "        4. Update parameters\n",
    "        param inputs: array of length sequence_length that represents the character indices of the inputs to\n",
    "        the network\n",
    "        param targets: array of length sequence_length that represents the character indices of the targets\n",
    "        of the network \n",
    "        return loss\n",
    "        '''  \n",
    "        \n",
    "        x_batch_out = self.forward(x_batch)\n",
    "        \n",
    "        loss = self.loss.forward(x_batch_out, y_batch)\n",
    "        \n",
    "        loss_grad = self.loss.backward()\n",
    "        \n",
    "        for layer in self.layers:\n",
    "            layer._clear_gradients()\n",
    "        \n",
    "        self.backward(loss_grad)\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GRUs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `GRUNode`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GRUNode(object):\n",
    "\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        param hidden_size: int - the number of \"hidden neurons\" in the LSTM_Layer of which this node is a part.\n",
    "        param vocab_size: int - the number of characters in the vocabulary of which we are predicting the next\n",
    "        character.\n",
    "        '''\n",
    "        pass\n",
    "        \n",
    "    def forward(self, \n",
    "                X_in: ndarray, \n",
    "                H_in: ndarray,\n",
    "                params_dict: Dict[str, Dict[str, ndarray]]) -> Tuple[ndarray]:\n",
    "        '''\n",
    "        param X_in: numpy array of shape (batch_size, vocab_size)\n",
    "        param H_in: numpy array of shape (batch_size, hidden_size)\n",
    "        return self.X_out: numpy array of shape (batch_size, vocab_size)\n",
    "        return self.H_out: numpy array of shape (batch_size, hidden_size)\n",
    "        '''\n",
    "        self.X_in = X_in\n",
    "        self.H_in = H_in        \n",
    "        \n",
    "        # reset gate\n",
    "        self.X_r = np.dot(X_in, params_dict['W_xr']['value'])\n",
    "        self.H_r = np.dot(H_in, params_dict['W_hr']['value'])\n",
    "\n",
    "        # update gate        \n",
    "        self.X_u = np.dot(X_in, params_dict['W_xu']['value'])\n",
    "        self.H_u = np.dot(H_in, params_dict['W_hu']['value'])        \n",
    "        \n",
    "        # gates   \n",
    "        self.r_int = self.X_r + self.H_r + params_dict['B_r']['value']\n",
    "        self.r = sigmoid(self.r_int)\n",
    "        \n",
    "        self.u_int = self.X_r + self.H_r + params_dict['B_u']['value']\n",
    "        self.u = sigmoid(self.u_int)\n",
    "\n",
    "        # new state        \n",
    "        self.h_reset = self.r * H_in\n",
    "        self.X_h = np.dot(X_in, params_dict['W_xh']['value'])\n",
    "        self.H_h = np.dot(self.h_reset, params_dict['W_hh']['value']) \n",
    "        self.h_bar_int = self.X_h + self.H_h + params_dict['B_h']['value']\n",
    "        self.h_bar = tanh(self.h_bar_int)        \n",
    "        \n",
    "        self.H_out = self.u * self.H_in + (1 - self.u) * self.h_bar\n",
    "\n",
    "        self.X_out = np.dot(self.H_out, params_dict['W_v']['value']) + params_dict['B_v']['value']\n",
    "        \n",
    "        return self.X_out, self.H_out\n",
    "\n",
    "\n",
    "    def backward(self, \n",
    "                 X_out_grad: ndarray, \n",
    "                 H_out_grad: ndarray, \n",
    "                 params_dict: Dict[str, Dict[str, ndarray]]):\n",
    "        \n",
    "        params_dict['B_v']['deriv'] += X_out_grad.sum(axis=0)\n",
    "        params_dict['W_v']['deriv'] += np.dot(self.H_out.T, X_out_grad)\n",
    "\n",
    "        dh_out = np.dot(X_out_grad, params_dict['W_v']['value'].T)        \n",
    "        dh_out += H_out_grad\n",
    "                         \n",
    "        du = self.H_in * H_out_grad - self.h_bar * H_out_grad \n",
    "        dh_bar = (1 - self.u) * H_out_grad\n",
    "        \n",
    "        dh_bar_int = dh_bar * dtanh(self.h_bar_int)\n",
    "        params_dict['B_h']['deriv'] += dh_bar_int.sum(axis=0)\n",
    "        params_dict['W_xh']['deriv'] += np.dot(self.X_in.T, dh_bar_int)\n",
    "        \n",
    "        dX_in = np.dot(dh_bar_int, params_dict['W_xh']['value'].T)\n",
    " \n",
    "        params_dict['W_hh']['deriv'] += np.dot(self.h_reset.T, dh_bar_int)\n",
    "        dh_reset = np.dot(dh_bar_int, params_dict['W_hh']['value'].T)   \n",
    "        \n",
    "        dr = dh_reset * self.H_in\n",
    "        dH_in = dh_reset * self.r        \n",
    "        \n",
    "        # update branch\n",
    "        du_int = dsigmoid(self.u_int) * du\n",
    "        params_dict['B_u']['deriv'] += du_int.sum(axis=0)\n",
    "\n",
    "        dX_in += np.dot(du_int, params_dict['W_xu']['value'].T)\n",
    "        params_dict['W_xu']['deriv'] += np.dot(self.X_in.T, du_int)\n",
    "        \n",
    "        dH_in += np.dot(du_int, params_dict['W_hu']['value'].T)\n",
    "        params_dict['W_hu']['deriv'] += np.dot(self.H_in.T, du_int)        \n",
    "\n",
    "        # reset branch\n",
    "        dr_int = dsigmoid(self.r_int) * dr\n",
    "        params_dict['B_r']['deriv'] += dr_int.sum(axis=0)\n",
    "\n",
    "        dX_in += np.dot(dr_int, params_dict['W_xr']['value'].T)\n",
    "        params_dict['W_xr']['deriv'] += np.dot(self.X_in.T, dr_int)\n",
    "        \n",
    "        dH_in += np.dot(dr_int, params_dict['W_hr']['value'].T)\n",
    "        params_dict['W_hr']['deriv'] += np.dot(self.H_in.T, dr_int)   \n",
    "        \n",
    "        return dX_in, dH_in"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `GRULayer`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GRULayer(object):\n",
    "\n",
    "    def __init__(self,\n",
    "                 hidden_size: int,\n",
    "                 output_size: int,\n",
    "                 weight_scale: float = 0.01):\n",
    "        '''\n",
    "        param sequence_length: int - length of sequence being passed through the network\n",
    "        param vocab_size: int - the number of characters in the vocabulary of which we are predicting the next\n",
    "        character.\n",
    "        param hidden_size: int - the number of \"hidden neurons\" in the LSTM_Layer of which this node is a part.\n",
    "        param learning_rate: float - the learning rate\n",
    "        '''\n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        self.weight_scale = weight_scale\n",
    "        self.start_H = np.zeros((1, hidden_size))       \n",
    "        self.first = True\n",
    "\n",
    "        \n",
    "    def _init_params(self,\n",
    "                     input_: ndarray):\n",
    "        \n",
    "        self.vocab_size = input_.shape[2]\n",
    "\n",
    "        self.params = {}\n",
    "        self.params['W_xr'] = {}\n",
    "        self.params['W_hr'] = {}\n",
    "        self.params['B_r'] = {}\n",
    "        self.params['W_xu'] = {}\n",
    "        self.params['W_hu'] = {}\n",
    "        self.params['B_u'] = {}\n",
    "        self.params['W_xh'] = {}\n",
    "        self.params['W_hh'] = {}\n",
    "        self.params['B_h'] = {}        \n",
    "        self.params['W_v'] = {}\n",
    "        self.params['B_v'] = {}\n",
    "        \n",
    "        self.params['W_xr']['value'] = np.random.normal(loc=0.0,\n",
    "                                                        scale=self.weight_scale,\n",
    "                                                        size=(self.vocab_size, self.hidden_size))\n",
    "        self.params['W_hr']['value'] = np.random.normal(loc=0.0,\n",
    "                                                        scale=self.weight_scale,\n",
    "                                                        size=(self.hidden_size, self.hidden_size))        \n",
    "        self.params['B_r']['value'] = np.random.normal(loc=0.0,\n",
    "                                                       scale=self.weight_scale,\n",
    "                                                       size=(1, self.hidden_size))\n",
    "        self.params['W_xu']['value'] = np.random.normal(loc=0.0,\n",
    "                                                        scale=self.weight_scale,\n",
    "                                                        size=(self.vocab_size, self.hidden_size))\n",
    "        self.params['W_hu']['value'] = np.random.normal(loc=0.0,\n",
    "                                                       scale=self.weight_scale,\n",
    "                                                       size=(self.hidden_size, self.hidden_size))\n",
    "        self.params['B_u']['value'] = np.random.normal(loc=0.0,\n",
    "                                                      scale=self.weight_scale,\n",
    "                                                      size=(1, self.hidden_size))\n",
    "        self.params['W_xh']['value'] = np.random.normal(loc=0.0,\n",
    "                                                       scale=self.weight_scale,\n",
    "                                                       size=(self.vocab_size, self.hidden_size))\n",
    "        self.params['W_hh']['value'] = np.random.normal(loc=0.0,\n",
    "                                                       scale=self.weight_scale,\n",
    "                                                       size=(self.hidden_size, self.hidden_size))\n",
    "        self.params['B_h']['value'] = np.random.normal(loc=0.0,\n",
    "                                                       scale=1.0,\n",
    "                                                       size=(1, self.hidden_size))\n",
    "        self.params['W_v']['value'] = np.random.normal(loc=0.0,\n",
    "                                                       scale=1.0,\n",
    "                                                       size=(self.hidden_size, self.output_size))\n",
    "        self.params['B_v']['value'] = np.random.normal(loc=0.0,\n",
    "                                                       scale=1.0,\n",
    "                                                       size=(1, self.output_size))    \n",
    "        \n",
    "        for key in self.params.keys():\n",
    "            self.params[key]['deriv'] = np.zeros_like(self.params[key]['value'])\n",
    "        \n",
    "        self.cells = [GRUNode() for x in range(input_.shape[1])]\n",
    "\n",
    "\n",
    "    def _clear_gradients(self):\n",
    "        for key in self.params.keys():\n",
    "            self.params[key]['deriv'] = np.zeros_like(self.params[key]['deriv'])\n",
    "                    \n",
    "        \n",
    "    def forward(self, x_seq_in: ndarray):\n",
    "        '''\n",
    "        param x_seq_in: numpy array of shape (batch_size, sequence_length, vocab_size)\n",
    "        return x_seq_out: numpy array of shape (batch_size, sequence_length, vocab_size)\n",
    "        '''\n",
    "        if self.first:\n",
    "            self._init_params(x_seq_in)\n",
    "            self.first=False\n",
    "        \n",
    "        batch_size = x_seq_in.shape[0]\n",
    "        \n",
    "        H_in = np.copy(self.start_H)\n",
    "\n",
    "        H_in = np.repeat(H_in, batch_size, axis=0)      \n",
    "\n",
    "        sequence_length = x_seq_in.shape[1]\n",
    "        \n",
    "        x_seq_out = np.zeros((batch_size, sequence_length, self.output_size))\n",
    "        \n",
    "        for t in range(sequence_length):\n",
    "\n",
    "            x_in = x_seq_in[:, t, :]\n",
    "            \n",
    "            y_out, H_in = self.cells[t].forward(x_in, H_in, self.params)\n",
    "      \n",
    "            x_seq_out[:, t, :] = y_out\n",
    "    \n",
    "        self.start_H = H_in.mean(axis=0, keepdims=True)     \n",
    "        \n",
    "        return x_seq_out\n",
    "\n",
    "\n",
    "    def backward(self, x_seq_out_grad: ndarray):\n",
    "        '''\n",
    "        param loss_grad: numpy array of shape (batch_size, sequence_length, vocab_size)\n",
    "        return loss_grad_out: numpy array of shape (batch_size, sequence_length, vocab_size)\n",
    "        '''\n",
    "        \n",
    "        batch_size = x_seq_out_grad.shape[0]\n",
    "        \n",
    "        h_in_grad = np.zeros((batch_size, self.hidden_size))        \n",
    "        \n",
    "        num_chars = x_seq_out_grad.shape[1]\n",
    "        \n",
    "        x_seq_in_grad = np.zeros((batch_size, num_chars, self.vocab_size))\n",
    "        \n",
    "        for t in reversed(range(num_chars)):\n",
    "            \n",
    "            x_out_grad = x_seq_out_grad[:, t, :]\n",
    "\n",
    "            grad_out, h_in_grad = \\\n",
    "                self.cells[t].backward(x_out_grad, h_in_grad, self.params)\n",
    "        \n",
    "            x_seq_in_grad[:, t, :] = grad_out\n",
    "        \n",
    "        return x_seq_in_grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Single LSTM layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAD0CAYAAABtjRZ7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd2CUVdr38e+U9EZCSIFAKCGH3ntHZAFdVOyKFF0pLrbdx13XVVfXhfV59rWiLioCgmJFRMUOSgfpJZQTSgKEEJKQnpA+7x8zCQkEEmCSyZ1cn7+YkpmT0fxyct3nXMdks9kQQghhTGZXD0AIIcTVkxAXQggDkxAXQggDkxAXQggDkxAXQggDs9bVGymlPIC+wGmgpK7eVwghDM4ChAPbtNYFFz5YZyGOPcDX1+H7CSFEQzIU2HDhnXUZ4qcBli5dSlhYWB2+rRBCGFdSUhITJ04ER4ZeqC5DvAQgLCyMiIiIOnxbIYRoEKosQ8uFTSGEMDAJcSGEMDAJcSGEMDAJcSGEMDAJcSGEMDAJcSGEMDBDhHhRSSmjXl7D+sMprh6KEELUK4YIcRNw/GweW46ddfVQhBCiXjFEiFstZsKbeJKQfs7VQxFCiHrFECEOEO7vRVJmvquHIYQQ9Uq12+6VUhZgPqAAGzDT8XVvA8VALPCg1rpUKTUNmOG4f7bWeqWzBurhZia3oNhZLyeEEA1CTWbi4wG01oOBZ4A5wHPAC1rrIYAHcKNSKgx4FBgMjAFedLSfdQqL2URxqRzqLIQQFVUb4lrrFcB0x81IIAPYBQQppUyAH1AE9AM2aq0LtNaZwBGgm7MGajWbKC6REBdCiIpqVBPXWhcrpRYDbwBLgcPAXOAgEAqsAfyBzApflg0EOGugVrOZEpmJCyFEJTW+sKm1ngJEY6+PzwWGaq07AEuAl4Es7LPyMn7YZ+1OYbGYKCotddbLCSFEg1BtiCulJimlnnLczANKgTTsoQ2QCAQCW4GhSilPpVQA0BGIcdZA3cwmmYkLIcQFanIoxHJgkVJqHeAGPA6cBT5RShUDhcA0rXWSUmou9iPYzMDTWmunrQm0mM1SExdCiAtUG+Ja61zgzioeGlzFc+djL7c4ndVsoljKKUIIUYlhNvtYLVJOEUKICxknxM0miqScIoQQlRgmxC2yxFAIIS5imBB3s0hNXAghLmSYELfIjk0hhLiIYULc6uidYrNJkAshRBnjhLjFPlQpiwshxHmGCXGL2QTYj2oTQghhZ5gQd7PYQ1xWqAghxHmGCXGL2T5U6SkuhBDnGSbErY5ySrGUU4QQopxxQlzKKUIIcRHjhHjZhU0JcSGEKGeYEC+riZfIhh8hhChnmBAvW50iW++FEOI8w4R42TpxWZ0ihBDnGSbEz69OkRAXQogyBgpxR01cZuJCCFHOMCFusZStTpGauBBClDFMiJeVU2QmLoQQ5xkoxB3b7qUmLoQQ5YwT4rLEUAghLmKcEJclhkIIcREDhbjs2BRCiAsZJsTPb/aRcooQQpSxVvcEpZQFmA8owAbMBJId9wUCFmCy1vqoUmoaMAMoBmZrrVc6a6Dnt93LTFwIIcrUZCY+HkBrPRh4BpgD/AdYqrUe5rivg1IqDHgUGAyMAV5USnk4a6AW2bEphBAXqTbEtdYrgOmOm5FABvagjlBKrQImAmuAfsBGrXWB1joTOAJ0c9ZArXKyjxBCXKRGNXGtdbFSajHwBrAUaA2ka62vB04ATwL+QGaFL8sGApw10POHQkhNXAghytT4wqbWegoQjb0WngF87XjoG6APkAX4VfgSP8fznKL8UAgppwghRLlqQ1wpNUkp9ZTjZh5QCqwDbnDcNwzYD2wFhiqlPJVSAUBHIMZZA7XItnshhLhItatTgOXAIqXUOsANeBzYDbynlHoIewnlXq11ulJqLrAe+y+Hp7XW+U4bqEVq4kIIcaFqQ1xrnQvcWcVDo6t47nzs5Rank9PuhRDiYobZ7GOVdeJCCHER44S4HAohhBAXMUyIO6opUk4RQogKDBPiJpMJN4tJyilCCFGBYUIc7MsMJcSFEOI8Q4W41WyW3ilCCFGBsULcYpJt90IIUYGxQtxsokjKKUIIUc5QIW4xm+RkHyGEqMBQIW41m+XCphBCVGCsELeY5Hg2IYSowFAhLksMhRCiMkOFuJvZLDVxIYSowFAhbp+JSzlFCCHKGCrEZdu9EEJUZqgQt5hN0sVQCCEqMFSIW81miqSLoRBClDNWiFtM0jtFCCEqMFiIm2XbvRBCVGCoEHe3mCkslnKKEEKUMVaIW01SExdCiAqMFeIyExdCiEqMFeJWWZ0ihBAVGSrE3WQmLoQQlVire4JSygLMBxRgA2ZqrWMcj90LPKK1Hui4PQ2YARQDs7XWK505WHermUKZiQshRLmazMTHA2itBwPPAHMAlFI9gT8AJsftMOBRYDAwBnhRKeXhzMFKTVwIISqrNsS11iuA6Y6bkUCGUqop8G/g8QpP7Qds1FoXaK0zgSNAN2cOVmriQghRWbXlFACtdbFSajEwAbgDWAD8GThX4Wn+QGaF29lAgJPGCdhr4qU2KC4pxWoxVDlfCCFqRY2TUGs9BYgGVgDdgXnAJ0AnpdRrQBbgV+FL/IAM5w3VPhMHKJKt90IIAdQgxJVSk5RSTzlu5gFJQCet9QjgbuCA1vpxYCswVCnlqZQKADoCMc4crLtj9i11cSGEsKvJTHw50FMptQ74EXhca33uwidprZOAucB64Bfgaa11vjMH6+aYiReUlDjzZYUQwrCqrYlrrXOBOy/xWDwwoMLt+diXI9YKD4uUU4QQoiJDXR10s5oAKacIIUQZQ4W4u8UCIMsMhRDCwVghbpULm0IIUZGhQtzN4iinyExcCCEAg4W4zMSFEKIyY4V4+eoUCXEhhACjhbjMxIUQohJDhbib7NgUQohKDBXi5TNxKacIIQRgtBCXmbgQQlRirBCXLoZCCFGJsUK8fCYuDbCEEAIMFuIebo4uhlJOEUIIwGAh7uVmwWSC3IJiVw9FCCHqBUOFuMlkwtfdSk6BlFOEEAIMFuIAPh5WcgqKXD0MIYSoFwwY4hZyZSYuhBCAAUPc18NKjtTEhRACMGKIe1rlwqYQQjgYLsR93GUmLoQQZQwX4lJOEUKI8wwX4j4eUk4RQogyBg1xWZ0ihBBgwBD387RSWFJKgfRPEUIIrNU9QSllAeYDCrABMx1f9wZQAhQAk7XWZ5RS04AZQDEwW2u90tkD9nG3AJBbUIKH1eLslxdCCEOpyUx8PIDWejDwDDAHeB14RGs9AlgOPKmUCgMeBQYDY4AXlVIezh6wj4f9947UxYUQogYhrrVeAUx33IwEMoC7tda7HfdZgXygH7BRa12gtc4EjgDdnD1gX0eIywoVIYSoQTkFQGtdrJRaDEwAbtdanwZQSg0CHgaGYZ99Z1b4smwgwLnDtW/2AQlxIYSAK7iwqbWeAkQD85VSPkqpu4C3gRu11ilAFuBX4Uv8sM/ancpHZuJCCFGuJhc2JwERWusXgTygFLgVe4llhNY6zfHUrcAcpZQn4AF0BGKcPWBfqYkLIUS5mpRTlgOLlFLrADfgcWARcAJYrpQCWKu1fk4pNRdYj32G/7TWOt/ZAy6fiedLiAshRLUhrrXOBe684O6gSzx3PvbliLVGLmwKIcR5htvs4+9pxdfDysm0PFcPRQghXM5wIW4ymYgK8eVwco6rhyKEEC5nuBAHaB/iS+wZCXEhhDBkiEeH+pGaUyAlFSFEo2fIEL++UygWs4kPfzvu6qEIIYRLGTLE2wT70DsykJV7TnOuULoZCiEaL0OGOMBjo9pzKuMcn20/6eqhCCGEyxg2xAdHBdMx3J9v95529VCEEMJlDBviAIPaNWXvqQxKSm2uHooQQriEoUO8Y7g/+UWlxKXmunooQgjhEoYO8U7h/gAcPJ3l4pEIIYRrGDrEo0J8cbOYOCAhLoRopAwd4u5WM1EhfjITF0I0WoYOcYCO4X4cSJQQF0I0ToYP8U7h/iRnF5CaU+DqoQghRJ1rECEOcnFTCNE4GT7EO0qICyEaMcOHeKCPO+EBnlIXF0I0SoYPcbDPxg+eznb1MIQQos41iBBvG+zD8bRcbDbZfi+EaFwaRIi3DPImv6iUFFmhIoRoZBpIiHsBcDLtnItHIoQQdathhHigNwAJ6cY9rq24pJSCYjngQghxZayuHoAzRDhC3Ihnbh5NyeHPn+7mSHIONuCuvi15/PpoArzcXD00IYQBVBviSikLMB9QgA2YCeQD7ztuxwCztNalSqnngBuBYuBxrfXWWhp3JV7uFpr5eXDCYCF+LCWH2+dtotQGv+/WnJzCYpZsPs7PB87w+cyBhAd4uXqIQoh6riYz8fEAWuvBSqkRwBzABDyjtV6jlHobuFkpdRwYDvQHWgJfAH1rZdRVaBnoVe9q4jabjU1Hz+Lv6UaIvwfb4tOIDvUjOtSPYyk5TF20DZPJxFd/HETrYB8AdhxP5575W3j5p1heuqO7i78DIUR9V22Ia61XKKVWOm5GAhnA9cBax33fA78DNPCT1toGnFBKWZVSzbTWKbUw7ou0DPJmx/H0unirGrHZbMz59iDvbYirdL+bxcT4bs1ZdfAMVouZhVP7lgc4QO/IQCYNiOT9TfE8Nqo9LYO863roQggDqVFNXGtdrJRaDEwAbgdGO8IaIBsIAPyBsxW+rOz+OgnxFk28+HbvaUpKbVjMprp4y8v6ZNtJ3tsQx919W9I7MpCcgmJUmB8L1sexfNcphrYP5t8TulYZ0g8ObcOSzfEs2BDH8zd1rvTYgcQsvtt3mqSsfG7u0Zyh7ZvV0XckhKiPanxhU2s9RSn1JPAbULFY64d9dp7l+PeF99eJFoFeFJfaSM7Od3ktubTUxrvrjtEtIoA5E7pW+qUyqF0w+UUleLpZLvn14QFe3NS9BZ9uO8kfR7YjxM8TgM1HzzLxvS2U2sDH3cKyHQnMGN6Wp8Z1rPXvSQhRP1W7xFApNUkp9ZTjZh5QCmx31McBxgHrgY3AGKWUWSnVCjBrrVNrYcxVigyylySOJOfU1Vte0u6EDOJSc5kysHWVfxVcLsDLzBzelhKbjQlvbWLu6sMs25HA9A+2E9nUh9/+Poodz47m7r4teWftMekbI0QjVpN14suBnkqpdcCPwOPALOCfSqnNgDuwTGu9A3uYb8Z+UXNW7Qy5aj1aNcFiNrE1Lq0u37ZKa3QKZhOM6hhy1a/RPtSPT6cPINjXnVd+juWJz/cQ5OPOhw/2J9TfE083C38d28H+frHJzhq6EMJganJhMxe4s4qHhlfx3OeB5695VFfB18NKVDNf9teDWelanUyPlk1o4u1+Ta/Ts1UgXz08hNScAk6ln6NjuD/u1vO/d4N83An29SA+NfdahyyEMKgGsWOzTKfm/uw+meHSnY+pOQXsSchkpLr6WfiFgn096N6ySaUAL9Mm2Jv4VGOtjxdCOE+DCvFberYgLbeQnw+ccdkY1sXaF+OMcGKIX07rpj7EnZWZuBCNVYMK8SFRwYT6e/DwR7v475ojlJTWfmvaklIbL/2o+dsXe0nNKWCNTiHY153Ozf1r/b0BWgf7kJJdQE5BcZ28nxCifmkQvVPKWMwmft+tOQs2xPGfHzSHTmfzr1u61Gofki92JvDmr0cA+wXNpKx87urTEnMdrVVv49goFJ+aS5cWAXXynkKI+qNBzcQB/jw6mtm3dOHhkVF8szeRWUt31uphEcu2JxAd6ssn0weQlV+Em8XE1MGta+39LtS6qSPEpaQiRKPUoGbiAD4eVu4bEAlAsK87z39zgJV7TzO+e3Onv1dBcQm7TqbzwJA2DGjblNX/M5yCotJK2+hrW+tg+45PWaEiROPU4GbiFU0a2JpuEQE8syLG3urVyTPy2KQcikpsdGvRBLDvtKzLAAfwdrcS6u9BnKxQEaJRatAhbjGbeOOenpwrLOH6V9Yy7vX1HEpy3jryY6n23aFRIb5Oe82r0bqpj5RThGikGnSIA0Q29WHh1L7c1L05abmF3PH2ZqcdHhGXmovJBJFNXdtpsE2wj5RThGikGnyIAwxpH8zce3ryxUODKCgu5b9rjjrldeNTc2ke4FWjXii1qXWwD2dzC8nKL3LpOJwtLbeQH/cn1clSUSGMqlGEeJmWQd7c0TuCL3YkkJyVf9nnnsnKJ/ZMNvlFl979GXc2r/zCoiuVr1BpQLPx3IJibnpzAzM+2MFHW0+4ejhC1FuNKsQBpg9rS3FpKQs2xl3yOZ9uO8HAF1fzu1fXMfQ/v7Lh8MXNGG02G3EpOeUB6kpla8XjGlCIz/nuIKcy7Cc1vfZzLNkN7K8MIZyl0YV4ZFMfbugazkdbTlRZfjiTlc/TX8YwsF1TXrmzO0283Lj//a1sOlI5yNPzisjKLy4PUFeKbOqNu8XMvoRMVw/FKXaeSOej307wh8Ft+GrWYM7mFvLO2mOuHpYQ9VKjC3GAmcPbkV1QzIdbjle6Pykzn6mLtgEw55au3NorgmUPDaJ1Ux8e/WQX+xMzy+uzZbPe+hDinm4W+rUJYm1snRyi5FTJWfmMf2MDt/53I3NXH+a7fad5dkUMIX4ePD46mu4tm3BT9+bMX3+MhHRZRinEhRpliHdpEcDQ9sHMX3eMpMx8ftXJ3PD6ega8uJoTZ3NZUOHcywAvN968txfZ+cXcOHcDkxb8BpyvP9f1uvBLGaGacTg5h2/2JLLrRLpheqm8uiqWfacyiT2Twys/x/LHpTs5cDqL52/qjK+HfS/aX8cqLGYTL35/yMWjFaL+aXA7NmvqoRHtuHf+bwx4cTUA7Zr5MLF/K+7p1+qiHiQqzI/vHxvKP77az4YjqZzNKSD+bC5mE7QMdP2FTYDbe0fw9tpjPPLxLgDaBvvw3WNDXb5y5nKy8otYtiOBif1b8bdxHUjPLeJUxjnCAjwr/YUTEejNpIGRzF93jLjU3Hrx14+4cjabjZ0n0vn1UAqJmecY2LYpv+scVt7bKDkrHw83S632OmqIGm2ID2oXzDuTenMgMQubzcZDI6Lwcr904LVt5svD10Wx4UgqexzHr0UEelfZ49sVmni78/1jQ4k5lcnBpCz+84Pm+5jTTOgZ4eqhXdKO+HSKSmzc2DUcP083/DzdaHWJNfd/GNKGDzYf58XvDvLu5D51PFJxrc7mFDDn24Ms33UKi9lEgJcby3ee4ukVMVzfMYS8whLWxqYQ5O3OykeHuPycXCNptCEOMKZzGGM6h9X4+d0iAjCbYHt8OnGpufWmlFKmmZ8HIzuEMDy6GZ9sPcln2xLqdYj/FpeG1WyiZ6vAap8b4ufJrJFR/L8fNY98vIvfdQplWHQzmbXVc+m5hTyzIoZv950G7NejHrkuCm93C3sSMlmyKZ71R1LJzi/i7r6t+HJXAk9+sY/F9/fFZKqbTqBG16hD/Ep5u1sZ1C64fLPQo9dFuXhEVTObTdzZJ4KXfopl54l0etUgJF1he3waXVoEXPYvoIoeHNqGmFOZfLMnkW/2JBLs68EXDw0ksh4s8xQX230yg4nzt5BfXMqM4W25vmMofVsHlT/eo2UTetzVg5JSG8WlpXhYLahQX57/5gDf7D3NTbXQtK4hqh+1AAN5+saOBPnYD314cFhbVw/nku4bEEnLIC+mL9nB6cxzrh7ORQqLS9l3KpPekTX/BeNhtTDvvt4cnjOOj6cNoLi0lKmLtpGWW1iLIxVXo7TUxnNf78fHw8qKPw7mqXEdKwV4RRazCQ+r/Rd5WdO6F745QEae/HetCQnxK9Qx3J+NT17H1w8Pwd+z/v4p38TbnYVT+pJfVMK0Jds5V+iac0d3HE8n5tTF69cPJWVRUFxKz1ZNrvg13SxmBrZrynuT+3Aq4xwPLt7Gu+uOMn3JdvaczKjRaySk53Hrfzfy4vcHr/j9RfW+3pPInpMZPDm2A10jan5YicVs4t8TupJ5rpCJ7/3mtD5HDZmE+FXwcrdgqaOTe65F+1A/5t7Tg/2JWUz/YDvrD6fUWn+VzHNFvL7qMIs3xZe3KkjOyue2eZv4/RsbLmpzsPN4OsA1lXr6tA7i9bt6sPNEBv/+7hA/HTjDHW9v5osdCQB8vv0kf/p0Nyv3JlJcUlrpa99YfYSdJzJ4Z+0xYs9kX/UYxMVOZ57j+W/20y0igAk9W1zx13dpEcC7k/twLCWX615ew9Nf7pO/ti5DauIN3HUdQnnh5i7873cHmbRgK1aziRdu7sK9/Vtd8WvZbDbS84oI9HbjZNo5PtgST4CXG70iA3n6y5jyDVDfx5xmxrB2PLMipvxrF22K58mxHcpv7zqZQai/B+EBntf0/Y3rGs7b9/WmoLiEYe2bMeujnTyxbA+fbDvBtnj7L4ovd52ia4sAPnywPwFebpzKOMfyXQmM6RzKj/vPsEYnEx3qd03jEOe9+nMseQUlvHZXj6s+pnCkCuHXJ0bw5q+H+XTbSdYdTmHJA/1leWkVJMQbgUkDIrm1Zwu2H0/nvfXHeGbFPvYmZODnacXLzYINe08Zv8uUh7Lyi5i1dCfrD6cS5u9JcnY+FZsLBni58fnMgZxMy+Ovy/ay5dg2WgV58+n0ASzZfJwlm+IZ1K4pQ9s3Ayi/4OqMFQhju5xfYTR/ch/ufncL2+LTuaN3BLMndOGHmCSe+HwPD7y/jUX39+VvX+zFYjbx7O87cSwll/WHU5k+rN01j0PA6oNn+Gx7Ag8OaUPbZtfWZz8swJPZt3Tltl4R/GHxdu5ftJWVjw4t3wQm7OTTaCR8PKwMj25G39aBTF6wlU+2ncRiNlFqs2GzwaajZ5k3sRch/pVnxqcyzvF/3x9ix/F0zmTl88DgNhw/m8sNTcOZOqg1FouJlXsSGdUxhKgQP/q2DiLM35OfDpxh+rC2NG/iRUSQN/pMNlMWbuW9KX3oFB7AybRzTHIco+fs7/OzGQM5kpxTXou9uUcL3C1mHvl4F92e/wmAf93ShYhAbwZHBfPJthMUFpfWmzX/RpVXWMyTX+ylY7g/fxmrnPa6PVsFMm9iL+6ev4WXf9I8N76z0167thQUl5CRV0So4+cpNaeAUpuNEL9r+8uzKpcNcaWUG7AQaA14ALOBE8DbQDEQCzyotS5VSk0DZjjun621Xun00Ypr5u1u5ZPpA/gtLo3ekYGYTSZ+3J/EX5ftZfLCrSz/4yC83e3/W9hsNp5ctpdNR1MZ1C6Y/9zejcFRwRe95ozhlWexg6KCGVTheS2aePHVrMHc9e5mHvpwJxGB9o0cI1RIrXyPXu6Wiy6mjesazgIPKy9+d5AxncO4z1FO6tcmiPc3xROTmFlvl2Iaxde7E0nNKWTefb3LV5s4S/+2TbmvfySLN8VzS48WdG955RfE68qJs3nc9e5mTmfmc0+/lvzr5i786dPdeFgtvDfF+RvVqpt63Aec1VoPBcYCbwLPAS9orYdgD/YblVJhwKPAYGAM8KJSysPpoxVOYbWYGRwVjKebBXermfHdm/POpN7oM9k8+vGu8iZfP8QkseFIKv/4fSc+fLB/lQFeUz4eVhZN7ccNXcOJS83l9t4RdV6HHh7djB8eH8afRkeXl3HKlr1ti0ur07E0NDabjcWbj9MhzI8+V7Bs9Er8Zawi2NeDp5bvu+hCdX2RnV/E9A+22/vhd2/Ox1tPMvvbg2yPTy+fvDhbdSH+OfCs498m7LPsXUCQUsoE+AFFQD9go9a6QGudCRwButXKiEWtGBbdjOfHd2bVwWTe/OUIeYXF/GvlATqE+XGfk8oezfw8ePWuHujZ43jpju5Oec1r1czPgzbBPmyLlxC/FhuPnOXg6SweGNym1nZa+nu68c+bOnPgdBYLNlz6PABXen3VYQ4n5/DWxF7MvacnkwZE8v6meM4VlTDkGiZBl3PZcorWOgdAKeUHLAOeAWzAW45/ZwJrgNsd/y6TDdR8caioFyYPjGTPyQxeWx3LqoNnSMzM57W7e2K1OLdW7Obk17tW/VoH8cP+JEpLbVe9mqKx+3DLcYJ83Lm5Z+3ushzbJYwRqhnz1h5lyqDW9arBW0ZeIR9vPcH4buHlF/D/Mlahz2QTHuDJyA61Uz6s9qdJKdUS+BX4QGv9EfA6MFRr3QFYArwMZGGflZfxA2q260LUGyaTidkTuhDVzJd9pzKZ0LMF/dpUvcuuIenbJojMc0UcTs4B7KWBz7afZMrCrTJDr4HkrHxWHTzDHb0jnF4Lv5DJZOLBIW3JyCvih5ikWn2vmrDZbHy37zRv/XqEWR/tJL+4lIdGnG/H4e/pxmczBvL63T1rbW9JdRc2Q4GfgIe11qsdd6dhD22AROx18K3AHKWUJ/Y6eUcgBmE4ZRc+Vx9K5sau4a4eTp3o56iLb41PQ4X5sWTzcZ77ej/uVjNb49L4fObAi9oT12dvrD7M2tgU/n5jxzq5WPvptpMUl9q4q2/LWn8vgEHtmtKiiRdf70nklqvYTORMX+9J5LFPdgNgMsFjo9qjwur2Wk91Swz/DgQCzyqlymrj04BPlFLFQCEwTWudpJSaC6zHPrt/Wmt9+ZOIRb3V1NeDO/vUzQ9kfdAyyIuIQC8Wb4onOSufRRvj6d8miDfu7cnv527gfz7bw3tT+tAyqH70jr+cA4lZvPxzLABTFmxl8R/61WqQF5WU8uFvxxnaPvia14XXlNls4sZu4SzaGEd6biGBPu518r5ljiTn4O9lJdjHgzd/OUJ0qC9fPzwEN4vZJTu5TTabrfpnOYFSqjUQt3r1aiIi6m97VNE4fb79JH9Zthewtxx+5c7uRIX48atOZtbSndhsMDiqKff0a0WovyfHz+ZxQ9ewetcudfbKAyzeHM9Xs4bw0NIdnMnK5617ezGqY2itvN/KvYk8/NEuFk7tw3Udauc9qhJ7JpvfvbqOMZ1DeWpcR1oFedfJ9YyFG+J4YeUBPKxmfDyspOUW8vrdPbi5R+39RZCQkMCoUaMA2mit4y98XDb7CIH9ZCQ/TyvNm3jRLeL8GuSRKoTvHh3KO+uO8uP+M6w6mFz+2IzhbXlqXEdXDLdKhcWlrNh9ipEqhE7N/fnioUE88P42pi3Zzj9v6sy9/SOdPlP8cMtxIjNjYEgAAAxrSURBVJt6MyK6di7aXUp0qB9/Hat45adYftx/hu4tm/D5jIG1umHLZrPx7rpjRIf60qV5AMt3naJ3ZCDju7m2Za6EuBDYL5iN7VL1NYDWwT68eGs3/ja2iAUb4/B2t7AvIZN31h7DbDLxwOA2NPNz/baIH/YnkZpTyD2OjUzBvh58PG0ADy3dybNf7WfZjgSeu6nzFZVXCovt67GrCsfkrHy2xqUxa2SUS1b1/HFEFOO7NWfJ5njmr49jbWwKozvV3l8D8WfzSMrK57Hru3JPv1Y8Mqo9Qd7uLl/RVL/WeglRjwV4u/Hn0dHMHN6O1+7uwS09mjNvzVFGvbyGHcddt4olIT3PHtBfxdA+xJfhjuVtYN9ktXBKH/5zWzcOJWVz6383MeOD7aTmFFT7umtjU+g9+2euf2Ut+xMrtxPOLyrhiWV7MZtM3NrLdeXRlkHe/GVMB3w9rPxyKLn6L7gGZW2Oy9ontwn2IcDb9e2oZSYuxFVws5h57e6ezBoZxdRF23h2xX5WPjKkTmdlB09n8cTne9ifaF8sFh3qyzuT+lw0BqvFzJ19WzK6UygfbzvBa6sOM+719bw3uc8lt68fS8lh5gc78PO0kpJdwI1zNzBSNWNc13AS0vL4PiaJw8k5/O+tXV3eWdDdamZYdDC/HDqDzdal1q5T7E/MxN1qJqqOLuDWlIS4ENegfagfT4yJ5k+f7uH7mCRu7Fb7yzJjz2Qzd/VhfjmUjL+nG0/f0JEerZrQtUXAZTe/BPq488cRUYxUIUxbsp3JC7dyc4/mpOUW8qfR0bSrEE7/+/0hLGYTKx8ZgofVwpLN8by/KZ5fdQpmk70p1Vv39qqT77cmRqoQvtuXxP7ELKcuB31/YxzfxyTx3PjO7E/MokOYn9M3v10rCXEhrtFN3VvwxuojvLvuaK2uWCkuKeXVVbEs2BCHm9nMyA4h/P2GjrRocmU9OTqG+/P+/f244+1NLNl8HLCfwLTo/r50CPNnjU7mpwNneOJ30eVdLR8Z1Z6ZI9qRkH6OQG83mnjX7bK+6oxQIZhM8MuhZKeFeEJ6Hv9ceQCbDe56ZzPZBcXc0+/K+/DXtvr1K0UIA7KYTdw/pA17EjLZ4TixyNkSM85x//vbeOvXo4zuFMbqJ4bz1r29rjjAy0SF+LLxb9dx8IWxfPfoUEptNu57bytLNsfz6Me7UKF+/GFI5TNk3Sxm2gT71LsAB3sPnO4RTZxaF/9272lsNpg3sRfZBcUA3NbLtZuLqiIzcSGc4LZeLXjpR83ba48yIauAmMRMPtt2kuyCYvw93ejXJpB9pzKxmEyE+nvSIcyPp2/sVKMlcRl5hdz05gZyC0r494SuV3UqU1XKWg53au7P0gcHcOc7m/nHV/tpG+zDgql98HKvP31JauK6DiG8uiqWlOwCp6wWWnXwDJ3C/RnXNZxVfx5GUmYBfS5x2LMrSYgL4QTe7lbu7d+KeWuOlq8lH9o+mE7N/YlLyWX3iQxaB/tgNplIzDzHb5vTyCss4f/VoJvjvLVHOZtbyFezBldaw+5MUSG+fDVrMDtPpDNChRDg5fpVF1fqug4hvPJzLGtjU7i997WtmEnLLWTH8XQevq49AFEhfkSF1M8j/CTEhXCSmcPakZJdwMC2TRkaHUwzX49L1sdf+lHz5q9HuKFbOCMvczhGclY+izfFM6FHi1oL8DItg7wN0VrgUjqF+9PMz4M1OvmaQ3zDkVRKbTCqljoPOpPUxIVwkgBvN166ozu39Y4gxM/zshc4HxkVRfsQX55evo/s/KJLPu/jrScpKC7lsevb18aQGxSz2cTw6GasP5x6zYdG7EvIwN1qplNzfyeNrvZIiAvhAh5WC/93ezeSsvKZumgbJ9PyLnqOzWbjy10JDGzblMimcsp7TYxUIWSeK2LzsbPX9Dr7TmXSMdy/3vW+r0r9H6EQDVSvVoHMvacnh05nMfrVtXz024nyo/EAdp7IIP5sHhNc3G7VSEZ1DCHEz95d8GqVltrYfyqLLgaYhYOEuBAu9ftuzVn1P8Pp2zqIv3+5j+tfWcuJs/ZZ+Ze7EvB0MzOukfR1dwZPNwsPjWjHb3FpfLDl+FW9xom0PLILiulqkB7yEuJCuFh4gBfv39+PV+/qTmpOAdMdvU1W7j3N7zqF4esh6w+uxMT+kYxQzXh2RcxVBfmeBHuPFKMcBCIhLkQ9YDGbmNAzgjfu6cmR5Bz6/3s1GXlFTKiHm0vqO3ermfcm96F3ZCAL1h+jpmcm7DieTlxqLmt1Ck283egYLuUUIcQVGqFCeOmO7piAHi2bMLSWTkhv6KwWM3f1bUn82Tx2naz+uN/v9p3mtnmbuO7lNSzfdYoxncJcckrP1ZC/04SoZ27p2YKRHULwcrPUu2ZLRjKuSxjProhhxa5Tl+2hnlNQzD++iiE8wBMfDyulpTYeGRV1yefXNxLiQtRDRtwxWd/4eboxulMo3+xJ5JnLtDhYuuU4qTmFrJg1mB6XaM1bn8mveSFEgzWhZwvS84pYF5tS5eM2m41Pt5+kT2SgIQMcJMSFEA3YsOhmNPVxZ97ao1Xu4vxk20mOpeRy34BIF4zOOSTEhRANlpvFzD/Gd2LH8XRufmsjf/5sN7862tUmZ+cz59uDDGrXlJt7uPaw42shNXEhRIN2c48WlNpszF8Xx0/7z/D17kQWTu3LD/uTKCi2t/etrYM86oKEuBCiwZvQM4IJPSPIyi/izrc3M3nhVgCmDmpNaxefEXqtLhviSik3YCHQGvAAZgNbgPlAIGABJmutjyqlpgEzgGJgttZ6ZS2OWwghrpi/pxuLH+jHbfM2kV9UwiPXGWcp4aVUNxO/DzirtZ6klAoCdgO/AEu11p8ppUYCHZRSucCjQB/AE9iglPpZa11Qm4MXQogrFervyY+PDwPApwG0NKjuwubnwLOOf5uwz7IHAxFKqVXARGAN0A/YqLUu0FpnAkeAbrUyYiGEuEY+HtYGEeBQTYhrrXO01tlKKT9gGfAM9tJKutb6euAE8CTgD2RW+NJswBjdY4QQwsCqXWKolGoJ/Ap8oLX+CDgLfO14+BvsJZQsoOIBdH5A9Q0LhBBCXJPqLmyGAj8BD2utVzvu3gDcAHwADAP2A1uBOUopT+wXQDsCMbU1aCGEEHbVFYX+jn0VyrNKqbLa+BTgPaXUQ9hLKPdqrdOVUnOB9dhn909rrfNra9BCCCHsLhviWuvHgMeqeGh0Fc+dj33poRBCiDoi2+6FEMLA6nKNjQUgKSmpDt9SCCGMrUJmWqp6vC5DPBxg4sSJdfiWQgjRYIQDRy+8sy5DfBswFDgNlNTh+wohhJFZsAf4tqoeNNX0EFEhhBD1j1zYFEIIA6v3zQOUUmbgv0B3oAB4UGt9xLWjqn2X6CB5AHgfsGHfTDVLa12qlHoOuBF7b5vHtdZbXTHmuqCUCgF2YF/mWkwj/jyUUk8BNwHu2H9G1tIIPw/Hz8pi7D8rJcA0GtH/G0aYid8CeGqtBwJ/A1528XjqSlkHyaHAWOBN4BXgGcd9JuBmpVQvYDjQH7gbeMtF4611jh/Wd4Bzjrsa7eehlBoBDMLekG440JLG+3ncAFi11oOAF4A5NKLPwgghPgT4AUBrvQV7r5bGoKoOkr2xz7YAvgeux/75/KS1tmmtTwBWpVSzuh5sHXkJeBtIdNxuzJ/HGGAf8CX2HkYrabyfRyz278uMvRlfEY3oszBCiF/YIbFEKVXvy0DX6hIdJE1a67Ir0WWdIhtFB0ml1FQgRWv9Y4W7G+3nAQRjn9DcAcwElgLmRvp55GAvpRzCvmt8Lo3o/w0jhPiFHRLNWutiVw2mLlXRQbLicd1lnSIbSwfJB4DRSqk1QA9gCRBS4fHG9nmcBX7UWhdqrTWQT+VAakyfx5+wfxbR2K+dLcZ+naBMg/4sjBDiG7HXvFBKDcD+J2SDV6GD5JNa64WOu3c5aqEA47A3HNsIjFFKmZVSrbD/kkut8wHXMq31MK31cK31COwnTE0Gvm+snwf2bqJjlVImpVRzwAdY3Ug/j3TOz7DTADca0c+KEcoSX2KfgW3CXhu+38XjqStVdZB8DJirlHIHDgLLtNYlSqn1wGbsv5RnuWS0rvE/wPzG+HlorVcqpYZhbwNd9n3G0Tg/j1eBhY7v0x37z852GslnIZt9hBDCwIxQThFCCHEJEuJCCGFgEuJCCGFgEuJCCGFgEuJCCGFgEuJCCGFgEuJCCGFgEuJCCGFg/x/RNijospq/9QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sd\n",
      "D Sls yase.shsnpaRUSoTA, :hecn sndaOrd\n",
      "AKfwy :wte tr nnah F\n",
      "OnTU,a o senineh?d dEC\n",
      " SL\n",
      "lo Lea, ct\n",
      "NRTaye: gh s sed  t\n",
      "r\n",
      ":Ad Eowt lce pehn raF\n",
      "Ede Rd randaqbhcaCdfye?noN oe bhe ete ketuaFanrNlows sH\n"
     ]
    }
   ],
   "source": [
    "layers1 = [LSTMLayer(hidden_size=256, output_size=62, weight_scale=0.01)]\n",
    "mod = RNNModel(layers=layers1,\n",
    "               vocab_size=62, sequence_length=25,\n",
    "               loss=SoftmaxCrossEntropy())\n",
    "optim = AdaGrad(lr=0.01, gradient_clipping=True)\n",
    "trainer = RNNTrainer('input.txt', mod, optim, batch_size=3)\n",
    "trainer.train(1000, sample_every=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Three variants of multiple layers:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD0CAYAAACPUQ0CAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhU1f3H8fdkgbAkLLLLKssXsLKJIEIABUXApfpTK+IGFbXVWrS1bijaoljrUtdqcUErVOterYiyY9gVUBAO+yLIvgVCQpb5/XEnk8kCiTDJZMLn9Tw+zD333Mk3E+eTkzP3nuvz+/2IiEjFERPpAkREJLwU7CIiFYyCXUSkglGwi4hUMAp2EZEKJi7SBZhZZeAs4CcgO8LliIhEi1igIbDQOZcRuiPiwY4X6rMjXYSISJRKBr4ObSgPwf4TwIQJE2jQoEGkaxERiQrbtm1j6NChEMjQUOUh2LMBGjRoQOPGjSNdi4hItCk0ha0PT0VEKhgFu4hIBaNgFxGpYBTsIiIVjIJdRKSCUbCLiFQwUR3sb8/byA2vL4h0GSIi5UpUB/uq7al89+O+SJchIlKuRHWwx/h85OgGUCIi+UR1sAPk6NZ+IiL5RHWwx/h8oFwXEcknyoNdI3YRkYKiO9hjNMcuIlJQVAe7TyN2EZFCojvY8aFcFxHJL6qDPcYHfn16KiKST5QHu+bYRUQKKtEdlMysHvANcD6QBYzHO9FwGXCbcy7HzEYDgwP7RzrnFphZq6L6hqt4nRUjIlJYsSN2M4sHXgEOB5qeBkY555IBH3CpmXUB+gDdgauBF4/WN5zF+3zeHLtf4S4iElSSqZgngZeBrYHtM4GZgceTgP5AL+BL55zfObcJiDOzukfpGzYxPh+APkAVEQlxzGA3sxuBnc65ySHNPudcbpSmAjWAJGB/SJ/c9qL6hk0g1zUdIyISorg59uGA38z6A52At4B6IfsTgX3AgcDjgu05RbSFTUwg2BXrIiJ5jjlid871ds71cc71BZYA1wOTzKxvoMtAYDaQAgwwsxgzawrEOOd2AYuL6Bs2vsCQXSN2EZE8JTorpoA/AOPMrBKwAnjfOZdtZrOBuXi/LG47Wt8w1BykOXYRkcJKHOyBUXuuPkXsfxh4uEDbqqL6hkuM5thFRAqJ6guU8j48jWwdIiLlSVQHe95UjJJdRCRXVAd73oenES5ERKQciepgD57uqBG7iEhQlAe7RuwiIgVFdbDrylMRkcKiPNh1HruISEFRHeyaYxcRKSzKg11z7CIiBUV5sHv/ao5dRCRPVAe7Dy0CJiJSUHQHe3COPbJ1iIiUJ1Ed7FrdUUSksOgO9kD1mooREckT3cGuG22IiBQS1cGuRcBERAqL7mAP/KsLlERE8kR1sAc/PI1wHSIi5UmUB7v3r+bYRUTyRHWwB+fYcyJciIhIORLVwa4Ru4hIYXHFdTCzWGAcYHjT2bcGjnsZyAJWATc553LMbARwS6B9jHPuMzOrA0wEqgBbgWHOubRwFK9le0VECivJiP1iAOdcT2AU8CgwGvizc64XUBkYbGYNgDuAnsAAYKyZVQYeAiY655KBxXjBH57ic5cU0MenIiJBxQa7c+5j4ObAZjNgH15A1zYzH5AIZALdgBTnXIZzbj+wBugA9AK+CBw/CegftuJ1HruISCElmmN3zmWZ2ZvA88AEYDXwHLACqA/MAJKA/SGHpQI1CrTntoWFbo0nIlJYiT88dc7dALTBm29/Dkh2zrUF3gKeAg7gjd5zJeKN7kPbc9vCIm8RMAW7iEiuYoPdzK4zs/sCm2lADrAHL7DB+0C0FrAASDazBDOrAbQDlgEpwKBA34HA7HAVnzdiD9cziohEv2LPigE+BN4ws1lAPDAS2A28Y2ZZwBFghHNum5k9hxfcMcADzrl0MxsDvBk4Y2YXcE24ig/OsSvZRUSCig1259wh4KoidvUsou84vKma0LbtwIXHW+CxBG+0URpPLiISpaL8AiUt2ysiUlCFCHbluohInigPdu9fjdhFRPJEdbDrrBgRkcKiPNh1HruISEFRHeyaYxcRKSzKg937V3PsIiJ5ojzYtQiYiEhBUR3suTRiFxHJE9XBrjl2EZHCojvYA9XrrBgRkTzRHeyaYxcRKSTKg937V3PsIiJ5ojrYwUv2eet28+2mvRGuRUSkfIjqYM8dsU+Yv4nLX5oT2WJERMqJKA92X6RLEBEpdxTsIiIVTFQHe8Fcz9bpMSIi0R3sMTH5k/3QkawIVSIiUn5EdbAXnIg5mK5gFxGJ6mAvOMd+KEPBLiISV1wHM4sFxgEG+IFbgR2BtlpALHC9c26tmY0AbgGygDHOuc/MrA4wEagCbAWGOefSwlF8fGz+YE9VsIuIlGjEfjGAc64nMAp4FHgCmOCc6x1oa2tmDYA7gJ7AAGCsmVUGHgImOueSgcV4wR8W8XH5y9eIXUSkBMHunPsYuDmw2QzYhxfejc1sCjAUmAF0A1KccxnOuf3AGqAD0Av4InD8JKB/uIqvFJu/fM2xi4iUcI7dOZdlZm8CzwMTgObAXudcf2ATcA+QBOwPOSwVqFGgPbctLOILBrtG7CIiJf/w1Dl3A9AGb259H/DfwK5Pga7AASAx5JDEQL/Q9ty2sIgtcLqjgl1EpATBbmbXmdl9gc00IAeYBQwKtPUGlgMLgGQzSzCzGkA7YBmQEtJ3IDA7fOXnp6kYEZGSjdg/BDqb2SxgMjASuAu43szmABcCjznntgHP4QX3NOAB51w6MAa42sxSgB7AC+H+Jn7VtQmV42J0VoyICCU43dE5dwi4qohd5xfRdxzeVE1o23a88C8VGx4fDMDUlTtITc8srS8jIhI1ovoCpVBJCXEc0FSMiEjFCfbEhDhSFewiIhUp2OM1FSMiQgUK9ppV49l76EikyxARibgKE+x1qldm10EFu4hIhQn2A4czOZiRxWLd1FpETnIVJtgrBRYEm7Jie4QrERGJrAoT7PcObAtAzSqVIlyJiEhkVZhgr1ElnspxMew8mBHpUkREIqrCBLvP56NO9crsTFWwi8jJrcIEO0DtapXYmxb+M2PSM7PJyfGH/XlFREpDhQv2PWE+lz0jK5vOf/6KMf9bEdbnFREpLRUq2E8Jc7DvOXSEUR8t43BmNq+nrA/b84qIlKYKFeyn1qrCT/vTw3bDjQc/WcZ73/wY3H7qS8dn320Ny3OLiJSWChXs3VrUJjvHzy9GT+ZIVs4JPdeB9Ez+991P+dqen7aG2ycu5v/+MSff8/9jxlr++N7SIufhl27ex3S344RqERH5OYpdjz2atG2QFHy8btfBfNslNXXFdt6et5Hpbmew7dSaVdiy73Bw+5uNe5m5aif92tbjutfnk7JmNwB3nd+GRjWrALB5Txq3vv0Ny7ce8Op5bBAxBW7lJyJSGipUsNepnndx0oZdh44r2H/95qLg416t6vDm8G5kZGVz24Rv+dOFbVm8aR/3f/Q9I95aVOjYNTsOUrtaJfr8bTrbD+Q/7XL51gOc0Ths9/EWETmqCjUV4/PljYhXbz9YbP+dqRn5pk8270kLPv7zpafz9k3diY3xUbVSHG8M60a7hklc071poecZNbgdANe/voC2D34RDPUHBrXj09t7AfDE5JVMXr6NfaVwOqaISKgKNWIPtXDj0RcD++CbH5m4YBPfbNzLmc1q8cFvzgEIflD6xcjkY472p9zVh0c+Xc6Bw5lc2bUJQ7s3pW5iZX7/zpJgn5R7z+PUwLTMeW3rMW3lDmav3gXAgxe159e9Wpzw9ygiUpQKNWIHmPqHPrSqV51Zq3Zy+Eh2of1Z2Tn84b2lfBMI/m827uX1r9fz+KSVPDd1NYmV44qdwmlVrzr/+nV3Prm9F9ee3Qyfz8elnU6lfUPvuOeGdA6GOsC467vmO/4vn/1wot+miMhRFTtiN7NYvBtUG+AHbnXOLQvsuwb4nXOuR2B7BHALkAWMcc59ZmZ1gIlAFWArMMw5l1b4K4VHy7rVeWBQO4aNX8gTk1cy+uLT8+3/4acDwcdV4mM5nJnNn0OCdsAvGhz3137v1h7MXLWTQWc0zNceG+Pjw9+ew1tzNvDxEu90yQv/PovXbjwr3y8AEZFwKMmI/WIA51xPYBTwKICZdQZ+DfgC2w2AO4CewABgrJlVBh4CJjrnkoHFeMFfqpJb1wFgyeZ9APj9fv67dCs7UtO5+73vAPj4tp6s+MuFXHd2s+Bxd/RrzYOD2x/3161WOa5QqOfq0rQWf7+6M3cPMABWbkvloudmH/fXEhE5mmKD3Tn3MXBzYLMZsM/MTgEeA0aGdO0GpDjnMpxz+4E1QAegF/BFoM8koH+Yaj+quNgYhnRrwtodB8nJ8bNyWyp3/Hsxg56dzaodqZxaswqdmtQE4JFL8kb0Q7s3pUbV+FKtbXjPFsFfPHvTMnWfVhEJuxLNsTvnsszsTeB5vGmV14C7gNSQbknA/pDtVKBGgfbctlLXuWktDqRnsW7XQX7a752DvuvgEfz+/GEeE+Pj8i6nAt6SBKWtSqVY/vXr7jx22RkAvDJzXal/TRE5uZT4w1Pn3A1AG+BjoCPwD+AdoL2Z/R04ACSGHJII7CvQnttW6joHRuT9n57FPR98n29fp6Y1820/dtkZLHnofOJiy+6z5CHdmhAX4+OdhZvK7GuKyMmh2CQzs+vM7L7AZhqwDWjvnOsLXA384JwbCSwAks0swcxqAO2AZUAKMChw/ECgTCaWW9atHny8MzWD+Fgfyx8ZwNz7zqNO9cr5+ibEx1Kzatneecnn83FT8mnsOniENTuKP+deRKSkSjJE/RDobGazgMnASOfc4YKdnHPbgOfwgnsa8IBzLh0YA1xtZilAD+CFcBV/LDExPmJDLuHPzPZTrXIcDWuUn7NQru/hfXA7VfdpFZEwKvZ0R+fcIeCqo+zbAJwdsj0O79TI0D7bgQtPqMrjdO+FbXn0c28d9aKuGI20RjWr0PyUqoydtJKruzWlRpXS/eBWRE4OFfbKU4CbkltwXY9mZGTmUK1ybKTLKdLQ7s149PMVzFmzi67Na/Ptpr1c0L5+vuURRER+jgod7D6fj4T4WBLiy2eoA1zXoxljJ63gNxO+Dbb987ozueD0479QSkRObhVuSYFokxAfy8UdG+Vrm7tu93E9V3aOn6zsE1uHXkSin4K9HPjbFR15fkhnLgkE/MyQteBL6r1Fm2l5/+dc+crccJcnIlFGwV4OVIqL4eKOjXhuSGfuHdiWdbsOFbp3a06OnwNHuUp11qqd3P2+t1TC4k37+DqwiqSInJwU7OXMmc1qAdDlL18x6uPvyQlMrwx8djYdHv6S+UVM0zz83+UAjEj2lgJ+ZsqqIm/TJyInBwV7OdM1EOwAb8/bxB/fW8rsNbtw273VG371z3ls2HUo2Oe9RZtZF9i+63zjqq6N+WbjXl6YvqZsCxeRckPBXs74fD7m3ncej19+Bq3qVefDxVsY9sbCfH3+8tkP+P1+9qdlBqdgJtzUnSqVYnnssjPo1qI2T3+1imkrj37hk9+vD1pFKioFeznUsEYVru7WlNduyLtBxx3ntWL9WG9lhqkrd/DJkq28u8hbZ6ZD4xr0bOWtGBkXG8O5Vg+A4eMX0fze/7Ftf3qhrzH6v8tp9cAk5h3nGTgiUn4p2MuxZqdUY8Pjg/liZDJ39GuNz+fjyzt7A/C3yS643vwr152Z77gbz2lOx5AbZ3+8ZEuh535r7kYApq/cUVrli0iEKNijQNsGScGVJ9vUT2TU4HZs2XeYz7/fRqcmNQutf1OlUiyf3N6L927tQXysj+enrmb/Ye+MmukrdzAm5I5RoXeUEpGKQcEehYb3bMHgwJ2aOjQ++vL2ZzWvzQvXdOHQkWzenLMBgGHjF/Lq1+uDfZZvPYDfrzNoRCoSBXsUionx8cI1nXnhms7cP6jdMfsOOL0BdapXImXNrnznwY9IbsF9A9uy59ARVmvZYJEKRcEepXw+Hxd1aFSidXCu6daU+ev3cPd7SwH416+78cDg9vRrVx+Aa1+dX6q1ikjZUrCfBHLXopm83Dv98azmtQFoVc+7GcmO1IwiL3wSkeikYD8JhN5N6rLOp+Yb5d/RrzXgXfi062DGMZ8n7UhW6RQoImGlYD8JxMT46Gt1AS/YQ/VrWy/4eNU27+rWr37YzgMffc+RrLwLmKau2E77hyazbMt+RKR8U7CfJJ4b0pkHBrWjR8tT8rV3bFKTxQ+eT6W4GCYv30Z2jp8HPvqeCfM38dzU1exM9Ubx/1m0GUAXNIlEAQX7SSIpIZ4RvU8jPrbwj7xWtUokt6rDm3M30u6hL9gRCPMXpq/hrEenkJPjZ8s+7za3KwOjehEpvxTsAkCv1t6SBLnTL6c3Sgruu+mtRSzb4l3ItHjT3rIvTkR+FgW7ADCkW1N6t6lLjA9eu6Er/7sjmZV/8e5BPm3lDirFxXBN96as3XmI/WlFrwsvIuVDsfc8NbNYYBxggB+4NXDc80A2kAFc75zbbmYjgFuALGCMc+4zM6sDTASqAFuBYc65tNL4ZuT4JcTH8tbwboXa/jG0C/+at5G//l8HNu9JY+L8TSz5cR992tSNUKUiUpySjNgvBnDO9QRGAY8CzwK/c871BT4E7jGzBsAdQE9gADDWzCoDDwETnXPJwGK84JcoMfCMhkwccTZNalfFGiQCsFZXqoqUa8UGu3PuY+DmwGYzYB9wtXNuSaAtDkgHugEpzrkM59x+YA3QAegFfBHoOwnoH77ypSzVrlYJgD9/9gPN7/0fb8/bGOGKRKQoJZpjd85lmdmbeNMvE5xzPwGY2TnA7cAzQBIQepJzKlCjQHtum0Qhn88XvHUfwKiPl9F1zBSe+WoVGVnZEaxMREKV+MNT59wNQBtgnJlVM7NfAS8Dg51zO4EDQGLIIYl4o/vQ9tw2iVITR3TnhWs6B7d3Hczg2amreXbK6ghWJSKhig12M7vOzO4LbKYBOcDleCP1vs65dYF9C4BkM0swsxpAO2AZkAIMCvQZCMwOY/1SxirHxXJRh0Z0aVozX/vHi7dwMENLDoiUByUZsX8IdDazWcBkYCTeh6eJwIdmNsPMHnHObQOewwvuacADzrl0YAxwtZmlAD2AF0rh+5Ay9sQVHfnr/53B8kcGMLxnC7buT+cXoyfz1Q9Hv8+qiJSNYk93dM4dAq4q0Fz7KH3H4Z0aGdq2HbjweAuU8qlVverB1SEfurg9SVXi+PuU1UxdsZ3z29ePcHUiJzddoCRhMbJ/G7q3qM20lTvYl3Yk0uWInNQU7BI2t5/Xih2pGXT681fc+MYCnvlqFdNX7mDEW4s0/y5ShoqdihEpqV6t6lC9chwHM7KY4XYyw+0M7pvyw3Z+WWDJYBEpHRqxS9j4fD6WPTKAGlXiC+0b+e4SJszXBU0iZUEjdgm7l4Z2YfbqXRw+kkXLetVpXS+RIePm8cinPzC0ezP8fj8+ny/SZYpUWAp2CbuererQs1WdfG3Xnt2Udxdu5pcvppAQH8M7N/eIUHUiFZ+mYqRMDO/ZgsxsP0s272Peuj2s33Uo0iWJVFgKdikTLepUy7d97pMzdNMOkVKiYJcy4fP5mHJXH94YdlZwIbHLXpqjkbtIKVCwS5lpVa8651o9Xr2+a7DtT+8vjWBFIhWTgl3KXK1qlfj8jmS6t6jNwg17WbRhT6RLEqlQFOwSEe0bJXFJp0YAXPHyXOau3c3eQ0e44JmZ/O+7nyJcnUh0U7BLxFzcsVHw8ZBx85i4YBOrth/ktonf8vn3CneR46Vgl4hJSohnw+ODaVgjAYC/TXbBfXf8ezFb9h2OVGkiUU3BLhH35vBuwcfjh53FlLt6k5Xj542v10ewKpHopStPJeLa1E9k2h/60LR2VeJivbFGy7rVePXr9Zx+ahKXdW4c4QpFootG7FIunFa3ejDUAZ68siMAd767lJmrdh7tMBEpgoJdyqXOTWvRrbl3o64bXl+A3++PcEUi0UPBLuXWS9d24fRGSQB8sWzbMftu2HWII1k5ZVGWSLmnYJdyq071ynz423MA+M2Eb1m1PbXIfm/N3UDfJ2dw7wfflWF1IuWXgl3KtcpxsQzp1gSAifM3FZqS2XPoCA99shyADxdv4YbXFxR6Dr/fr1Mn5aRS7FkxZhYLjAMM8AO3AunA+MD2MuA251yOmY0GBgNZwEjn3AIza1VU3/B/K1JRjb28Axt3pzF+zgbGz9kQbD/7tNrc0KM5AJXiYjiSlcPMVTvZkZpOvcSEYL/R/13OW3M38p9belA/qTLNTqlGcfalHeFIVg71khKK7StS3pRkxH4xgHOuJzAKeBR4GhjlnEsGfMClZtYF6AN0B64GXgwcX6hvWL8DOSmMvvj0Qm3z1u3hNxO+BWDZwwO4e4AB8HnIkgQ5OX7eWbAZgKtemUufv83gx71pwf2HjnKT7f/7xxy6PTaV9MzssH0PImWl2GB3zn0M3BzYbAbsA84EZgbaJgH9gV7Al845v3NuExBnZnWP0lfkZ7EGiawfO4gbz2nOkG5NuG9g2+C+zk1rUikuht/2bUnDGgk8/OkPwcD+x8y1HMnOoU396sH+vf46nZwcP/vTMjl99GR6PzE939dasH4Pa3d6ywl/vXpXGXx3IuFVoguUnHNZZvYmcBlwBXC+cy53sjMVqAEkAbtDDstt9xXRV+Rn8/l8PHxJ3sj95t6nMXXFDro2rxXc/6cLjTvfXcpjn6+gRZ1qwWUKPr8jmXW7DnHBM7MAuOLlOdw3qB0Am/akkZqeydLN+2l2SlXufHdJ8Gvc/f5SFj90QVl9iyJhUeIrT51zN5jZPcB8oErIrkS8UfyBwOOC7TlFtImcMJ/PR//29fO1Xda5MV8u386E+ZuCbfcObEtcbAxt6ifyyW09ufTFFL7dtI8rX54b7HPpiyms25l3049b+pzGKzPXsTctk/TMbBLiY0v/GxIJk2KnYszsOjO7L7CZhhfUi8ysb6BtIDAbSAEGmFmMmTUFYpxzu4DFRfQVKTU3ntM8+PgfQ7twa5+Wwe2OTWoy6+5zCx0TGuoAQ85qyhvDzgK8qRmRaFKSEfuHwBtmNguIB0YCK4BxZlYp8Ph951y2mc0G5uL9wrgtcPwfCvYN8/cgkk+3Ft4Vq7ExPvq1q19of9NTqgYfL7i/H6+lrOeVmev4fb/WXHFmY5rU9vbXT0qgUmwMs1fvpHebumVTvEgY+CJ9qbaZNQfWT506lcaNtdiThMeaHalk53gfuhZl4vxNNKyZwLlW75jPc9lLKSzetI/3bu1B5yY1OZiRRUZWDvV1GqRE2I8//ki/fv0AWjjnNoTu0+qOUiG1qld0oOe6pnvTEj1Pcqs6LC4wH5/r1j4tuTfk7ByR8kJXnoocw2/PbUWnJjWL3PfyzLVanEzKJQW7yDEkxMcy9vIzaNcwiXduPptb+pzGlLv6kNy6DgDrdh0q5hlEyp6CXaQY7RomMen3yZx92incN7AdrepV5y+X/gKA+z/8PsLViRSmYBc5Ds0CZ9bMX79Hyw5IuaNgFzkOPp+Pcdd3BeDSF1LYtj89whWJ5FGwixyn89p6p0q67alFLhcsEikKdpHjFBvjY2T/1oAX7rNX696sUj4o2EVOwO/7tWZ8YOmBt+dtjHA1Ih4Fu8gJ8Pl89LV6XNShIZOXb+fdhZuKP0iklCnYRcKgd2tvLZl7PtDpjxJ5CnaRMLi8y6m0qFONqpViOZiRRU6OrkiVyFGwi4RBXGwMD19yOmlHsvnF6MncF7hwye/3M3v1TvanZQIUG/hvztnA59//dMw+IsXRImAiYdKrVZ3g43cXbebhS05n8vJtjAzckcnqJ+K2p/LvEWfTo+Up+Y7dn5bJ89NW8+rX6/O1v35jV3q3rktcrMZgUnIKdpEwiY3xMeOPffnPos28NGMtr8xay9+nrA7ud9tTAfjqh+30aHkKa3ceJDEhjnqJCYz6ZBmfLt1a6DmHj1/ExR0b4QO2H0hn7OVncFrd6oX6iYTSeuwiYZaT4+e0+z8Pbt/Zvw3WoDq3vv1tkf0/ua0n1702nwPpWbRrmMTTV3Vk/+FMdh88wj0ffMfBwI25c3VsUpNNuw/RuWktpq3cQe82dXlz2Fn4fL58/dbuPIjf7y92CWOJTlqPXaQMxcT4+PuvOjHy3SXc0KMZv+nbkkpxMWx4fDCzVu3k+gJXqV76YgoAd53fhiHdmlI3sXJwX7NTqnLR81/n6790s3fb4GkrdwAwa9VO3lm4mSHd8taYX7PjIP2fngl4U0Cf3N5T9209iWjELhIBE+ZvpHW9RK56xbuBR4wP1o0dXGTf+et20/SUquT44bpX53Na3Wq0a5jEfxZt5qkrO3Hta/MBWDr6AmpUiQdgwDOzglM/AHcPMH51VhNenL6GW/u01B2gKoBjjdgV7CIRdCgji8HPzeauC4xLOjY6rud45NPlvJGygSeu6MBVXZuQneOnZWAq6LPf9So04geY8ce+NK9TLbiddiSLhLhYYmJ8hfpK+XSsYNdH7SIRVK1yHDPuPve4Qx3goYva06hGAn96/zua3/s/uj06BYAh3Zryi1Nr8PK1ZxY6pu+TM+j5+DSe+WoV+9My6TpmCqfd/zmHCsznl5as7BxWbU/lpRlr2HPoyM86dtfBDF7/ej3ZulbgqI45YjezeOB1oDlQGRgDbAJeBrKAVcBNzrkcMxsB3BJoH+Oc+8zM6gATgSrAVmCYcy6twNdojkbsIidk7KQVvDJzXXA7xudNzSQmeFMz2Tl+cgfjd7//He9/82Owb6XYGI5k5wS3B53RgEd/eQa1qlUKa405OX7enr+R9g2TmLRsG6+FnNrZtkEi467vSuNaVdi6P52ej08DYPLI3gz4+ywAalSJZ9Lvkxkybh4bd6cx4abu9Aw5xfRkc9xTMWY2DOjonBtpZrWBJcA3wDjn3OdmNgF4B1gIfAV0BRKArwOP/wZ865wbb2b3AhnOuWcKfI3mKNhFTkhWdg7fbtrHln1prN+Vxq/OasKpNascte+TX67CbTvAdOetSNmlaU3qJyUwadk2wDsn/+2buoe1xrlrdzNk3LywPV+/tvX45/VdiT1Jp49O5KyY972NmDQAAAvXSURBVID3A499eKPxxUBtM/MBiUAm0A1Icc5lABlmtgboAPQCHgscPynwOF+wi8iJi4uNoVuL2kDtEvW9d2BbABZu2MPkZdu4Z2Bb4mNj6Pn4NLbsO8zXa3YxdcV2+rWrH7YaJ8wvvPrlU1d25MxmtRj66ny27DscbL+gfX06Na3JE184AKb9oQ+/fDGFA+neVFH7hklMXbmDlvd/zhNXdKBu9cr87t+LuazzqZzXth7nBtbKP1kdM9idcwcBzCwRL+BHAX7gxcDj/cAM4IrA41ypQA0gKaQ9t01EyomzmtfmrOZ5vww+/V0vUtMz6fO3GXz23U90aFyT8XPWU6NKPDf3blno+J/2H2bxpn0MOqMhfr+fiQs2sefgEfamZbL9QDoDz2jA4DMasnnPYT777ifaNkjk4UtOZ+qK7fy2b6vgdM+0P/bhmw172bgnjU5NatKuYRIAt/ZuGfxA94uRvRk+fiEj+7fhlOqVuPJl74yiP73/XbCef83byL/mbWTgLxrw0tAuhc7tP1kUex67mTUBPgJecs5NNLMdQLJzbrmZ3QY8BUzGG73nSgT2AQcCjw+HtIlIOVW7WiVqV6tE56Y1+WjxFj5avCW4r2GNKlwc+JA3Iyub73/czxWBcE1KiKNOYmXW7TyU7/n+9/1PbBl4mLGTVgJw/6B2nH3aKZx9Wv4lFSrHxXJOqzqcU6Ce0LN0GtWswhcjewe3F9zfj3s++C44nXRTrxZMdztYu/MQk5Zt45IXUth+IJ0Xh3bJ98vrZHDMYDez+sCXwO3OuamB5j14gQ3eB6I9gQXAo2aWgPchaztgGZACDALGAwOB2WGuX0RKwTktT2HxpvzjsN/9ezFdm9ciKSGes8dOJTU97wyaA+lZwWmSgnJD/Zbep9G7Td2w1VgvKYE3hnUjPTObjKwcalSJZ9RF7Vm78yD9nprJ91u8yYIrX57LlLt6n1RX4BY3Yr8fqAU8aGYPBtpGAO+YWRZwBBjhnNtmZs/hBXcM8IBzLt3MxgBvBs6Y2QVcUyrfhYiE1R8vMOJjYxg/ZwNf3dmH77fsY/j4RfQYO42aVePzhfrS0Rcw5rMf6NKsFld1bcIPWw9wRuMaZOf4eW/RZu798HseueR0bjineanUmhAfm++q2pZ1q3PX+W14c84GdgdOpXzw4+W8ekNXqlU+OS621wVKInJU2Tl+YmN8+P1+WtyXt/5Ny7rVGNyhEcN7Nqdm1fCeFhluQ/45j7nrdnNpp0Y8e3XnE3qunBx/ubmISxcoichxyT2V0OfzMf/+fsH2d2/pwV3ntyn3oQ7wyvVnkhAfwydLtrJsy/7iDziKvYeO0PGRL3n4v8vDWF3pULCLSInUT0pgw+OD2fD4YOpUr1z8AeVEUkI8H/22JwDXvjafzJCLsX6Or1ZsJzUji/FzNvDMV6uYt243H4d8uAyQmp5Jemb2Cdd8ok6OCScROam1a5jEHf1a89zU1bR+YBIf39aTuBgfjWpW4Z4PvuPO/m1o3yjpmM/x/qK8q3WfnbqaZ6d6a+3XrlaJ1vWrs/9wJje8voDtBzK48szGXN+jOY1rVeGF6WtIO5LFw5ecTuW4sllhU3PsInJSyMzOofUDk466f9of+hAfG0OT2lUL7duflkmXMV/x614tSG5dh+teW1DEMxTv2as7cWmnU9mRms7yrQc4147/QirNsYvISS8+Noaloy/gjvNaFbn/vKdmkvzE9EI3NgF4Y4636NgvO51Kcuu6vHPz2Qzv2YIHL2pfqO/4YWdxZ/82we16Ievrf7JkKwczsuj26FSGvbGQy19KKZWpG43YReSk5Pf7cdtTsfqJPP7FyuAiau0aJjEiuQV3/Wcp4C1fsHLbAc5rW59Xb+ha6Hl2pmYwZ+0uBpzegPTM7OAHynsOHWHD7kN0aVoLgHve/453F20udHzKvecddV2fY9F67CIixcjIysZGfXHU/bP/dG6R0zQltWZHKv2fnhXcfuGaznRrUZt6icd30xNNxYiIFKNyXCxT7spbsuDyzqfy8MXtOa1ONZ68suMJhTp4F071CCylsPjB87moQ6PjDvXi6KwYEZGAVvUSWT92EAczsoJr2d/Ys0VYntvn8/Hvm88Oy3MVRyN2EZEQPp8vGOrRSsEuIlLBKNhFRCoYBbuISAWjYBcRqWAU7CIiFYyCXUSkgikP57HHAmzbti3SdYiIRI2QzCy0ZGR5CPaGAEOHDo10HSIi0aghsDa0oTwE+0IgGfgJiPwK9SIi0SEWL9QXFtwR8UXAREQkvPThqYhIBVMepmKOi5nFAC8BHYEM4Cbn3JoI1BEPvA40ByoDY4DNwGfA6kC3fzjn3jWz0cBgIAsY6Zw7vtuw/PwavwUOBDbXA68Azwbq+NI590gkX08zuxG4MbCZAHQChgBP4r2WAKOB2WVZo5l1B/7qnOtrZq2A8YAfWAbc5pzLKepnerS+pVxfJ+B5vOnMDOB659x2M3sW6AWkBg67FIgHJgJVgK3AMOdcWrjrK6LGzpTwfRGh1/AdoEFgV3NgnnPuajP7BKgDZAKHnXMDy6q+4xXNI/ZfAgnOuR7AvcBTEarjWmC3cy4ZuBB4ATgTeNo51zfw37tm1gXoA3QHrgZeLIvizCwB8IXUMgx4GbgG7w3fPfCGi9jr6Zwbn1sf8A1wB95r+KeQumeWZY1m9ifgVbxfNABPA6MCP2cfcOkxfqaF+pZBfc8Cvwu8hh8C9wTazwQGhLyO+4GHgImB+hYDt4S7vqPU+HPeF2X+Gjrnrg68fpcB+4A7A11bA70CNQ8sq/pORDQHey/gCwDn3Dyg8K1NysZ7wIOBxz68UceZwGAzm2Vmr5lZIl69Xzrn/M65TUCcmdUtg/o6AlXN7Eszm2ZmvYHKzrm1zjk/MBnoTzl4Pc2sK3C6c+6feK/hcDObbWZPmVlcGde4Frg8ZPtMYGbg8STyXrOifqZF9S3t+q52zi0JPI4D0gN/hbUG/mlmKWY2PLA/+DqWYn1F1fhz3heReA1zPQI875z7yczqAzWBT83sazO7KOR7Ke36jls0B3sSsD9kOzvw5i9TzrmDzrnUwP+k7wOjgAXA3c653sA6vGmEgvWmAjXKoMQ0vCmNAcCtwBuBtoJ1lIfX8368NxXAV8DvgN5Adbzay6xG59wHeH965/IFfhHC0V+z3Pai+pZqfc65nwDM7BzgduAZoBre9My1eH9N/tbMOhSou9T+PyziNfw574syfw0BzKwe0A9vmgWgEt5fhr/E+yXwTKBPqdd3IqI52A8AiSHbMc65wnehLQNm1gSYDvzLOTcR+Mg5901g90dAZwrXm4j3515pWwW8HRgRrcJ7E9Uuoo6Ivp5mVhMw59z0QNPrzrl1gTfPJxT9GpZljaHzp0d7zXLbi+pb6szsV3jTbIOdczvxfoE/65xLc86lAtPw/oILrbvM6uPnvS8i8hoCV+BNU+Weer0NeNk5l+Wc24E3dWURrK9EojnYU4BBAGZ2NvB9JIoI/Kn2JXCPc+71QPNkM+sWeNwPb944BRhgZjFm1hQvlHaVQYnDCcxFm1kjoCpwyMxampkPbyQ/m8i/nr2BqYGv7wO+M7Pcm+CGvoaRqnGxmfUNPB5I3mtW1M+0qL6lysyuxRup93XOrQs0twFSzCw28CF/L+BbQl7Hsqov4Oe8L8r8NQzojze1Err9HoCZVQd+AayIYH0lErVnxeD9xj/fzObgzW0Pi1Ad9wO1gAfNLHeu/S68P9ky8X7j3+ycO2Bms4G5eL9Qbyuj+l4DxpvZ13if4A/HG21MwLvA4Uvn3HwzW0hkX0/D+/Mc55zfzG4CPjSzw8APwDi8Mz4iVeMfgHFmVgnvjf2+cy77KD/TQn1LszAziwWeAzbhvWYAM51zo83sX8A8vCmHt5xzy81sDPCmmY0AduF9kF4WfgM8X8L3RZm+hiGC/x8COOcmmdkAM5uH97653zm3y8wiVV+J6AIlEZEKJpqnYkREpAgKdhGRCkbBLiJSwSjYRUQqGAW7iEgFo2AXEalgFOwiIhWMgl1EpIL5f5sTrsuGLEBSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "efoce howepe drrelee:\n",
      "ghiP Atus nowhr ot sirMacegSice dourt, herwe canew dod intAes aags\n",
      "ract, MAst loss, domdsd, be Orwe hin I hinensan: toOt heele.\n",
      "A drate potes they ther cotwtwed discbesst theremu\n"
     ]
    }
   ],
   "source": [
    "layers2 = [RNNLayer(hidden_size=256, output_size=128, weight_scale=0.1),\n",
    "           LSTMLayer(hidden_size=256, output_size=62, weight_scale=0.01)]\n",
    "mod = RNNModel(layers=layers2,\n",
    "               vocab_size=62, sequence_length=25,\n",
    "               loss=SoftmaxCrossEntropy())\n",
    "optim = AdaGrad(lr=0.01, gradient_clipping=True)\n",
    "trainer = RNNTrainer('input.txt', mod, optim, batch_size=32)\n",
    "trainer.train(2000, sample_every=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD0CAYAAACPUQ0CAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXiU5b3/8fdkhxDWsCmrLF8EN0DAKquAiOjR1lZxwe2cWk/V/vR4qoJQsS7V1qUuVSvWpVZbj2vrAmIRBJFFAUUUvsgii4AQ1hBIyPb745mEQBISIMlkJp/XdXldM/dzz+Q7D85n7rmfZ+4nVFhYiIiIxI64SBcgIiJVS8EuIhJjFOwiIjFGwS4iEmMU7CIiMSYh0gWYWTLQB9gI5Ee4HBGRaBEPtAY+c/eckhsiHuwEoT4r0kWIiESpAcAnJRtqQ7BvBHj55Zdp1apVpGsREYkKmzZt4rLLLoNwhpZUG4I9H6BVq1a0adMm0rWIiESbUlPYOngqIhJjFOwiIjFGwS4iEmMU7CIiMUbBLiISYxTsIiIxJqqD/eV5a7jiufmRLkNEpFaJ6mBfv30vc1ZmoIuFiIjsF9XB3rR+Ern5hWTm5EW6FBGRWiOqg71eUjwAe/dp7TARkSJRHezJCUH5+/IKIlyJiEjtEdXBnhQO9hwFu4hIsagO9uSEYComJ09TMSIiRaI82DUVIyJysKgO9iQFu4hIKZVaj93MWgALgOFAHvACUAgsAa539wIzuxMYFd5+k7vPN7POZfWtquLjQiEA8nUeu4hIsQpH7GaWCPwZ2BtuehgY7+4DgBBwvpn1AgYB/YDRwJ/K61ulxQe5jnJdRGS/ykzFPAg8DWwI3+8NfBy+PRkYBvQHprp7obuvBRLMrHk5fatMfDjZ8wuU7CIiRQ4Z7GZ2FbDF3T8o0Rxy96IkzQQaAQ2BnSX6FLWX1bfKxMVpKkZE5GAVzbFfAxSa2TDgFOCvQIsS29OAHcCu8O2D2wvKaKsy8eE59gKN2EVEih1yxO7uA919kLsPBr4ArgAmm9ngcJeRwCxgNjDCzOLMrB0Q5+4ZwKIy+lYZTcWIiJRWqbNiDnILMMnMkoClwOvunm9ms4A5BB8W15fXtwpqLlZ0VkyBpmJERIpVOtjDo/Yig8rYPhGYeFDb8rL6VpX9I/bq+gsiItEnqn+gVHS6o0bsIiL7RXewx2kqRkTkYFEd7EVnxejgqYjIftEd7DorRkSklKgOdk3FiIiUFtXBvn8qJsKFiIjUIlEd7HHh6rWkgIjIflEd7EUj9kIFu4hIsagO9jidFSMiUkp0B7vOihERKSWqgz1eZ8WIiJQS3cGus2JEREqJ6mAvOitGI3YRkf2iOti1pICISGnRHeyaYxcRKSWqgz2kS+OJiJQS1cEOwahdvzwVEdkv+oM9FOLrDbvYlZ0b6VJERGqFqA/2ffkFzPAtXPLM3EiXIiJSK0R9sBf5esOuSJcgIlIrxEywi4hIQMEuIhJjEirqYGbxwCTAgELgOmA80CrcpQMw191Hm9k/gXQgF9jr7iPNrDPwQvixS4Dr3V2LAIiIVJPKjNjPA3D3MwgC/V53H+3ug4EfAzuAm8N9uwD93X2wu48Mtz0MjHf3AUAIOL8K6xcRkYNUGOzu/jZwbfhue4IgL3IX8Li7bzSzlkBj4B0z+8TMzg336Q18HL49GRhWJZWXYV+evgiIiFQ4FQPg7nlm9iLBCP2nAGbWAhjK/tF6EvAQ8CjQFJhtZvOBkLsX/YIoE2hUdeUfKCsnj6SEpOp6ehGRqFDpg6fufiXQFZhkZqkEAf+Ku+eHu2wCnnb3PHffDCwimJcvOYxO48ARf5XKzM6rrqcWEYkaFQa7mY0xs7Hhu3sIgrqAYEplcomuw4DXwo9pAJwALAUWmdngcJ+RwKwqqbwMmTn69amISGVG7G8CPc1sJvABcJO77yUYja8q6uTuk4HlZjYXmAqMc/cM4BbgLjObQzBd83oVv4ZiuzViFxGpeI7d3bOAi8po71FG201ltC0HBh1pgYdDUzEiIjH2A6XdOQp2EZGYCvZMBbuISPQH+9yxQ3njv08HIFNL94qIVO489tqsVaMUWjZMJiEupIOnIiLEwIgdgkvkNUhJ0By7iAgxEuwAaSkJOitGRIQYCvYGyYkKdhERYijY05IT2K1fnoqIxE6wN6yXwM69GrGLiMRMsDdNTWLr7pxIlyEiEnExE+zN05LZmrWPgoLCijuLiMSwmAn29AbJ5BcUsnprVqRLERGJqJgJ9o7pqQAs25gZ4UpERCIrZoL95DaNAdi0KzvClYiIRFbMBHvj+okkJcSxWcEuInVczAR7KBSiRVoymzN1ZoyI1G0xE+wQHEDN0CmPIlLHxViwJ7F1975IlyEiElExFezNUjUVIyISU8HepWUDMnbnsDlTB1BFpO6KqWA/KXzK49cbdkW4EhGRyKnwCkpmFg9MAgwoBK4DEoF3gW/D3Z5y91fN7E5gFJAH3OTu882sM/BC+LFLgOvdvaCqXwjAcc2DHyld/fxnrLzvHOLjQtXxZ0REarXKjNjPA3D3M4DxwL1Ab+Bhdx8c/u9VM+sFDAL6AaOBP4Uf/zAw3t0HACHg/Cp+DcWapSYV3162SaN2EambKgx2d38buDZ8tz2wgyDYR5nZTDP7i5mlAf2Bqe5e6O5rgQQzax7u+3H48ZOBYVX9IoqEQvtH6F9/r2AXkbqpUnPs7p5nZi8CjwMvA/OBX7v7QGAVcCfQENhZ4mGZQCMg5O6FB7VVm6SE4CV9sX5Hdf4ZEZFaq9IHT939SqArwXz7VHdfEN70FtAT2AWklXhIGsHovqCMtmrzyW1DaJGWzJfrFOwiUjdVGOxmNsbMxobv7iEI6jfNrG+4bSiwAJgNjDCzODNrB8S5ewawyMwGh/uOBGZV5Qs4WIu0FH52aht8UybZufnV+adERGqlCs+KAd4EnjezmQRnw9wErAMeN7NcYBNwrbvvMrNZwByCD4zrw4+/BZhkZknAUuD1Kn4NpfRs24S8gkJeW7CeMae1r+4/JyJSq1QY7O6eBVxUxqYzyug7EZh4UNtygrNlasyArukAzP42Q8EuInVOTP1AqUhyQjwXndqGOau26lJ5IlLnxGSwA/Ru34Sde3P5TpfKE5E6JmaD/eS2wfIC7y3eSF5+Ae9/tZGde3MjXJWISPWrzMHTqNSlRXDm5UMfLucj38yitTv4j5OP4bFLerJpZzZJCXE0LfFLVRGRWBGzI/b4uBCpSfEALFobnNP+ry83sDkzm9N+N41ed39ITp5OhxSR2BOzwQ5w3aBOpdr63jut+LaNn8IHX2+qyZJERKpdTAf7DWd25sObBwLQMT31gG2ntm8CwK2vL67xukREqlPMzrFDsChYl5ZpPH91H7q1SmPjzmyenL6Cxy7pSf2kBB6YsoxJM1eRl19AQnxMf8aJSB0S08FeZIi1AKB1o3o8e2Wf4vaOzVLJKyhk485s2jatH6nyRESqVJ0epnYMX5jj5XlrI1yJiEjVqdPB3qtdE7q1SmOqDqCKSAyp08EeHxdi1ImtWZWRRWa2frwkIrGhTgc7wAltgut+6ALYIhIr6nyw9zimIQBLNyrYRSQ21Plgb94gmYYpCfxp+kpdAFtEYkKdD/ZQKETWvnwydudw1XOfaZlfEYl6dT7YAa4bdBwAm3Zl6yLYIhL1FOzALcONmb8eAsBPnvxU8+0iEtUU7EBcXIh2zepTP7wa5NXPfxbhikREjpyCvYRvfns2A7qks2lXNlsycyJdjojIEVGwH+SCU44F4IEpyyJciYjIkVGwH+TC3m24sFcb3lu8kd05eZEuR0TksFW4uqOZxQOTAAMKgevCj3scyAdygCvc/QczexToD2SGH34+kAi8AtQDNgBXu/ueKn4dVery09rxxsL1vLXoe8ac1j7S5YiIHJbKjNjPA3D3M4DxwL3Ao8CN7j4YeBO4Ldy3NzDC3QeH/9sJ/AZ4xd0HAIuAX1TtS6h6p7RtTLdWaUx4ewl79+nyeSISXSoMdnd/G7g2fLc9sAMY7e5fhNsSgGwziwO6AM+Y2Wwzuya8vT8wJXx7MjCsqoqvLqFQiP8aEJzbPuqxWRGuRkTk8FRqjt3d88zsRYLpl5fdfSOAmZ0O3AA8AqSGt18OnA380sxOAhoCO8NPlQk0qtJXUE1+2rsNAKsysli1ZXeEqxERqbxKHzx19yuBrsAkM0s1s4uBp4FR7r4F2AM86u573D0T+Ag4GdgFpIWfJo1gxB8VnrysFwBrttbqQwIiIgeoMNjNbIyZjQ3f3QMUAD8hGKkPdvdV4W1dgdlmFm9miQRTMAuB2cA54T4jgaiZ2+jXsSkAc1dvjXAlIiKVV5kR+5tATzObCXwA3ERw8DQNeNPMZpjZXe6+FHgJmAt8DPzV3b8G7gFGm9ls4EfAE9XwOqpFswbJDLbm/PnjVcxbpXAXkehQ4emO7p4FXHRQc9Ny+v4B+MNBbT8QzLlHpfGjuvPx8o95cc539DuuWaTLERGpkH6gVIHOLRpwVveWLP9BB1BFJDoo2CuhY3oD1mzNIl9rtYtIFFCwV0LH9Prk5hfy/fa9kS5FRKRCCvZK6JjeAIDVW7MiXImISMUU7JXQMT0VgClLNkW4EhGRiinYKyG9QRKN6iXy9/lr2Za1L9LliIgckoK9EkKhEH+6NPgV6j3vfRPhakREDk3BXkl9OjYB4K1F30e4EhGRQ1OwV1JyQjw3D+tKYSGs0UFUEanFFOyHYVj3FgB8sS5q1jETkTpIwX4YurZMIykhjiXf76y4s4hIhCjYD0NifBw9jmmoEbuI1GoK9sPUt2NTFq3dQcbunEiXIiJSJgX7YTrnhNbkFRTy2eptkS5FRKRMCvbDZK3SiI8L8c3GXZEuRUSkTAr2w5SSGE+n5ql8s0HBLiK1k4L9CHRv3VAjdhGptRTsR6BnuyZs3JnNgjXbI12KiEgpCvYj8LNT25CSGMe/vtDyAiJS+yjYj0D9pAQGdGnOh9/8QIGuqiQitYyC/Qid3aMVG3Zma65dRGodBfsROrltYwDuflfL+IpI7ZJQUQcziwcmAQYUAtcB2cAL4ftLgOvdvcDM7gRGAXnATe4+38w6l9W36l9KzerQrD4A81ZvIysnj9TkCneliEiNqMyI/TwAdz8DGA/cCzwMjHf3AUAION/MegGDgH7AaOBP4ceX6lulryBCEuLjeOXn/QB44dPvIluMiEgJFQa7u78NXBu+2x7YAfQGPg63TQaGAf2Bqe5e6O5rgQQza15O35hweqd0erdvwhsL1lNYqIOoIlI7VGqO3d3zzOxF4HHgZSDk7kVJlgk0AhoCJdezLWovq2/M+HHPY1mVkcXqDF18Q0Rqh0ofPHX3K4GuBPPt9UpsSiMYxe8K3z64vaCMtpgxsEtzAGb4lghXIiISqDDYzWyMmY0N391DENSfm9ngcNtIYBYwGxhhZnFm1g6Ic/cMYFEZfWNGu2b16dKiAVOWbIp0KSIiQCXOigHeBJ43s5lAInATsBSYZGZJ4duvu3u+mc0C5hB8YFwffvwtB/et4tcQcUOPb8nTH69k6+4cmjVIjnQ5IlLHVRjs7p4FXFTGpkFl9J0ITDyobXlZfWNJv45NefrjlazKyFKwi0jE6QdKVaBT8wYALFqrRcFEJPIU7FXg2Cb1SEtO4L73l7ElU5fME5HIUrBXgfi4EI9d0hOAif/6OsLViEhdp2CvIkO6tWBAl3SmfL2J9dv3RLocEanDFOxV6IELTyIEjH3zq0iXIiJ1mIK9Ch3TuB5n9WjJrG8zmPivr8nLj/q1zkQkCinYq9htZ3cDgoXBOt8xmW1Z+yJckYjUNQr2Kta+WSrPjOlNQlwIgOc+WR3hikSkrlGwV4OzerRixX3nMKBLOk9MX8GclVsjXZKI1CEK9mr0s1PbAnDJpLm89vm6CFcjInWFgr0anXtia56/qg8AL81dozXbRaRGKNirUVxciCHdWnDL8K4sXr+TjmPf550vN0S6LBGJcQr2GnBxn7bFt2/8+yJ8U2YEqxGRWKdgrwEtGqYwd+xQ2jUNLoA94o8zue31xezL03nuIlL1FOw1pFWjFGb872D6dGgCwKufr+PTlRkRrkpEYpGCvQbFxYV49oo+HNs4uLLgEx+t0AFVEalyCvYa1qh+IrNvP5Of9DqWz9ds506tBikiVUzBHiG3jgiWHvjrnDXk5heQX6CRu4hUDQV7hLRqlMKvRxgAXe6YTKdx77Nu2x4278qOcGUiEu0U7BH0y8GdGNAlvfj+gN9Pp+9907Seu4gcFQV7BIVCISZdcSp/vPgU+nfeH/AT//VNBKsSkWiXUFEHM0sEngM6AMnAPcClQKtwlw7AXHcfbWb/BNKBXGCvu480s87AC0AhsAS43t11AndYSmI8F/Q8lgt6HgvAUzNW8sCUZdz86hc8cvEpEa5ORKJRhcEOXA5sdfcxZtYU+MLd2wGYWRNgOnBzuG8XoIe7lzwS+DAw3t1nmNnTwPnAW1X2CmLM1Wd04IEpy3hr0ff0at+EpPgQhYVQLyme+kkJDO/eMtIlikgtV5lgfw14PXw7BOSV2HYX8Li7bzSzlkBj4B0zawzc7+7vAr2Bj8P9JwNnoWAvV0piPJ/cNoT+D0xnwttLSm2fP24oLRqmRKAyEYkWFc6xu/tud880szSCgB8PYGYtgKEE0ywAScBDwAXAT4BHwn1CJUbwmUCjKn0FMahNk/r8sZxpmL73TWPeKq3vLiLlq8yIHTNrSzDKftLdXwk3/xR4xd3zw/c3AU+7ex6w2cwWAQaUnE9PA3ZUSeUx7oKex3LuSa0JhULk5OUTIsRFf57DV9/v5OJn5vLv/xlE5xYNivuv376HNk3qR7BiEaktKhyxh6dYpgK3uftzJTYNI5haKXn/tfBjGgAnAEuBRWY2ONxnJDDr6MuuGxLi44iPC1E/KYF6SfG8c2N/rhvUCYDfvb+UzOxctmTmcNmzc+n/wHRmLt+iJQpEpFIj9nFAE2CCmU0It40kGI2vKurk7pPNbISZzSUYpY9z9wwzuwWYZGZJBEH/OnLEbh/ZjS2ZObyxcD0nTpx6wLYrnpvPacc15YWr+5KSGB+hCkUk0kKRHuGZWQdg9bRp02jTpk1Ea4kWqzOyGPLgjOL7zdOS2ZKZc0CfP4/pzYgerRCR2LR+/XqGDh0K0NHdvyu5TT9QikId01OZO3YoAMO7t+SzO4bh95zN8ntG0rJhMgC/eGkBz89ezfzV2zQ9I1LHVOrgqdQ+rRql8N39o4rvJycEUy8zbx3CZZPm8fma7dz1TvAL1hev6cugrs0jUqeI1DyN2GNMckI8vxzS6YC2Bd9ti1A1IhIJGrHHoDO7tSwezQ99aAaffbc9whWJSE3SiD3GnX/KscxZtZVvfzjwAtpTv97EoD9MZ902rSQpEmsU7DFudJ+2JCXE8fyn3wHwXUYWT3z0Lde+tIA1W/cw4PfT2ZypNeBFYommYmJci4YpnHtSa16Zt5aOzVK59/2lpfr0vXcay+4+u/jc9627c/h8zXbO6t6SUChU0yWLyFHSiL0OuH1kcBm+olBvmprE3RecwPJ7Rhb36TZhCu8u3gDAjX9fxC9eWkD333zA7pxgzbdvf8jkmw27arhyETkSCvY6oEVaCnef34NQCO76jx4snDCcMae1Jykh7oBTJm94ZREdbn+PT1cGi4ztzc3nvMc/YcXmTIY/MpNzHpulBchEooCCvY4Y86MOrP7dKK48vUOpbfPHDaVT89QD2t7/1QCuOr0DqzOyGPbwzOL2i5+Zy959+Qc/hYjUIppjF1o0TGHaLYNZuHY7O/fkclzzVNo3S2XCud15IXzQdfyo45num5m9YiuvL1zPmNPaR7ZoESmXRuxSrFe7Jgzp1oL2zYLRe3xciFd+3o/bR3bjP/t35Nkr+gAw4e0lXP38/GqpIScvnyXf76yW5xapKzRil0M6vVM6p3cKLrRdLymef1x7GqOfmct030KH29+jVcMUZvx6cLmrSeYXFPLl+h2c0qYxcXEh7nn3G579ZDUAF5xyDLeN7MY/5q+jY3oq3Y9pyFmPBNM+7/2qPz2O0TVZRI6Egl0Oy2nHNePT28/k9Ps/AmDTrmy6TZjC/xvahbZN6zPqxNbUS9of8s/OWsXvJi/juPRUzj6hVXGoA7z9xQbe/mJDmX/njQXfK9hFjpCCXQ7bMY3r8eHNA9m0K5sxfwmmZB6d9i0A909eymOX9OShqcs5vVMzXpj9HQCrMrJ4csZKAB762cnc8tqXZT73sONbsjsnl+dmr6ZziwZc0retzqUXOUxaj12Oyr68At7+4ntufX1xuX3+dGkv3lr0PVk5eTx40ckc27geO/fk0rBeAm8u/J4RJ7SiQfL+McaarVkM+sOM4vtPXNqTc0865pB1zF+9jeZpyXRMTz1kP5FYofXYpdokJcRx0alt+e7+Ufx5TO/i9rNLXORjRI+WPHvlqfz92tM4tnE9ABrVTyQUCnFh7zYHhDpA+2apXH1Gh+L7N7yyiE07y1/2YHVGFhf9eQ5DHpzBHW99VUWvTCR6acQu1aawsPCoplG+WLeDWcu38NCHy4vb3r2xPyccu3/u/eDRPcBndwyjeVpycQ3b9+Tyy5cX8OjonrRsmHLE9YjUJhqxS0Qc7dz4KW0bc+PQLvTp0KS47dzHP+HTlRn872tf0uH294pDvWlqEu/c0B+APvf+m315BcxZuZWOY9+n190fMnfVNs7+48yy/oxIzNHBU6n1/u8XP2Ln3lzueW8pry9Yz6WT5h2w/fohnfj1iGA9nEb1Etm5N5eu4yeXep7te3L5LiOLDpqHlxinEbvUeqFQiMb1k3jwZydz69lGWnICrRulMKJHcL3XolAHWDRhOG2b1iu+f2a3Fowd2Y1/3XAGAIMfnEGP30whv0DXga3NsnPzy71W74rNu/nf174kO1dLW5RHI3aJKr8c3JlfDu5c7va4uBCzbj2Tm/6xiFM7NOXyMpY+yNqXT6dx79M8LZkTjmnIbSO70a1Vw0r9/Zy8fO55dyn9u6RzUptGtG5Ur+IHSaVk5eSxfvte/vjv5UxesomrTu/AxP/ocUCf3PwChj38MRBcyP1vc9dwZrcWXH1Gx0iUXGtVePDUzBKB54AOQDJwD7AOeBf4NtztKXd/1czuBEYBecBN7j7fzDoDLwCFwBLgencvKPH8HdDBU6khC9Zs48Kn5hzQVi8xnqV3n12px//0qU/5fM3+Sw2uvO8c4uOq7jz77Nx85q3exhmdmpFXUFjuL3pjRWZ2LidOnFru9j9d2ouRJ7RiT24+O/fmMnP5Fsa+WfrMp26t0nhmzKm0a1a/uG1fXgHrtu+hQ7PUKv03qi0OdfC0MiP2y4Gt7j7GzJoCXwC/BR5294eKOplZL2AQ0A9oC7wB9AEeBsa7+wwzexo4H3jrqF+VyBHo3b4pq393Dg9NXc4T01cAwfLEl06ay9/+sx9xcSGyc/M559FZDOveknHnHF/82PveX3pAqAN0Gvc+x6Wn8vDFp3BK28ZHVVtBQSHdJkwp1f7JbUP465w1/Gpol+JTQ/PyC0iIjyO/oJCCwkIS46NrVnXh2u08Nu1bNu4o+zTWDs3q893WPVz/ysJS25IS4khPTWJDiVNgl23KZOAfpvOXK0+lf5d0cvIK+Mus1cU/nHvl5/2Kl8aoCyozYm8AhNw908yaAZ8BHwBG8MHwLXATcDVQ393vDz9uEXAWwQdBG3cvNLPzgbPc/foSz98BjdglAgoLC9mcmUO/+6aV2+ev1/RlYNfmrNsWXEYQ4Ms7zyIpPo7jf7M/hBPjQ/ztP/vR77hmZf6dis4Q2rMvj/959UumfL2p3D7jRx3PmB+159F/f8uzs1Yz8T968MzMlXy3dQ+3nm2HnKKqLb7esJNurRpyym+nkpmdV9w+vHtLrhvUiRZpybRqlEJCXIj/+3wdt71RenQ+sGtzGiTH8/5Xm/j1COPTlRnMXnHo6wQkJcTx5n+ffsCpstHuUCP2Sp/HbmZpwL+ASQRTMovdfYGZ3QE0AXYQjOyfCvefCVwDzHT3Y8JtZwLXuPvlJZ63Awp2iaANO/YWr31TpG+Hpsz/bhsQnBc/5MEZ7M7J46nLejHyxNbF/XxTJtN9M098tILdOXk8fXkvRvRoxd7cfEKEuOudr/nHZ+sYd043rh3Yqdwanp21inveC65wtWjCcD5fs50tmTls2pXNY9O+LfdxJS2eeBYNUxIPaNuetY+E+BBPfLSCP89cxYAu6fz1mr4RWaZh3Ftf8cq8tVzYqw1vLFxf3D7jfweXe6bS9zv2khgfokVaCg9+4DwxfQUvXN2Hri3TmLb0By7t1754muW9xRtLjfBHndiaE9s04v7JywB4479/RO/2TY/6tTw01Xn8oxWkN0hm/rihxEVgqueog93M2hJMnzzp7s+ZWWN33xHe1h14HPgnkOLuvw+3LwKGA1+4e5tw2/nAcHe/ocRzd0DBLhG2eVc2C9dup02T+mTszmFgl+b86h+LeHfxxuI+55zYiicv613m4+et2srFz8wt9/kT40PcPvJ43ly4nscu6UnztGTSkhNYuSWLvfvyOe+JT4DyQ+7pj1cWhxMEo/eiD4LL+rXj5XlrOalNI/4VPpd/7qqtjC6nnt9feBIX9WlLYWEh67fvpW3T+jw7axUAX32/k/t+fCKpyRXP0n66MoOGKYmVGgWv2Ly7+KBnSRf2asODPzupyj5oftiVzdRvfqBn28a0bVKfRvWDD7oOt78HQFpKAl9NHHFEz70rO5eTJk7lxGMb8VWJpaVPO64pj13Sk315BbRpUv+Ax6zbtofF63cysGs6WzJz6JieWmWv9aiC3cxaAjOAG9x9WrhtHnBj+ODojQRz6q8CvycI8zbAO+5+spm9AzxUYo59uru/WuL5O6Bgl1po555cTv7t/gN78+8YSou08n+5+sd/L+eP/z5wdH1Sm0aMHXk8l0wqP/SLPHDhiVzcp12Z2/ILCpm3aivZefkMsRYHhENhYSEdx75fXGN8KETve/5d6jluPdv4/RQH4Ke92/D6gvWl+gD86szO3Dy8K6FQiHm68YkAAAmwSURBVN05eeTk5tOsQfIBtfz4ydksXr+TRvUS+ff/DGLttj3sys7lb3PW0OOYhlx2Wntycgto16w+c1ZuLX79JUfriyYMp0lqUoX7pSp8uiKDS58Nfv9w5Y/ac9f5Jxz2c0z9ehPXvrSg+H69xHj2HnTK5bK7z+b9rzZy5z+/5ozO6aWm1qpyyuxog/1R4GJgWYnmOwhCPBfYBFzr7rvMbCIwkuD8+Jvd/RMz60owfZMELAV+7u75JZ6/Awp2qaX+Pn8ts77dwmOje5JQiQOUBQWFZOflkxQfd0D/309ZVry6ZVnaNa3PzFuHHHGdRd8Y0lISGNqtRfFyyC9e05d2Tevz0bLNXHV6Bxau3c7Pnp5TwbPBpf3a0b11Q8a/vQSAuWOH0qpRSpnTHRVp36w+a7bu4ZozOvKb87qzOyePEFTqW0FV2rMvj+6/+QCAb+8dWeqA8wNTlvHUjJUsnDCcpmV84Ix98yv+Pn8tAHPGnknrRvXYvCubvoc4RnOwlMQ4lt09suKOlVAlc+zVRcEudUnG7hy2Z+2jUb1EkhLiWLx+J1c8N58f9zyWRy4+5aie+6Z/LCoO9HqJ8Xzz2xFlfu1/d/EG7n73Gyae14O+HZuyKiOL1o1S2Ja1jxv/vog1W/dU6u8VBXZljB3ZjV8MKv8YQ015ed4a7nhrCY9cfDLdWjWkW6s0Plq2GYD/fPHz4n6r7juHgsLC4g/ntVv3MPAP0+mYnso/rj3tgDWHNu/K5sGpzv99HnwTaZ6WzK+GdmHC20sY3actE84NPsx+8dICvli3g0dHn8LunDwKCgpJSojjZ73bHtEcvYJdpJYqLCxk8pJNnNE5nUb1Eit+wCHk5Rcw4PfT2bgzm/t/ciKj+5Y9rXMomdm5bNqZzfDwlaxapCXTuUUDPl25/6yTktMJe/YFPyqauXwLw45vSYf0VPbsy2Pcm19RUBjMra/dtoe544aWWsUzEio6b75I0TTL81f1YUi3FnSbMJns3AIu6duO3/3kxDIf8/K8NUxbupknLu1JvcR4dufkkVbiYPbqjCyGPDij1OMOdfD4UBTsInVEXn4BK7dkYa3Sjup5/vDBMr76fhd/vaYvEIxYH5iyjJYNU5hw7vGHdQDwaFf5rGp/m7umeIrpYAsnDKfX3R8e0Dbh3O7c/e43pCbFs/A3w0lOOPIfjXUa9/4By1lcN6gTt4/sdohHlE/BLiISVlhYyLuLN3JmtxZ8vmY7rRulsHLzbhrWS+SMzunMXL6FK54rfbH2T28/k2MaH90SErtz8li0djsDujQ/queBo//lqYhIzAiFQpx3cnBFrkFdg4Dt2nL/N5yBXZvz3f2j+HRFBlc9/xn78gsY1LX5UYc6QIPkhCoJ9Yoo2EVEynB653SW3zuSzZnZpX74Vdsp2EVEDuFQv12oraJr5SAREamQgl1EJMYo2EVEYoyCXUQkxijYRURijIJdRCTG1IbTHeMBNm0q/8oxIiJyoBKZWWqNg9oQ7K0BLrvsskjXISISjVoDB6wJXRuC/TNgALARyK+gr4iIBOIJQv2zgzdEfBEwERGpWjp4KiISY2rDVMwRMbM44EngZCAH+C93XxGBOhKB54AOQDJwD7AOeBcougDmU+7+qpndCYwC8oCb3L302qDVU+NCYFf47mrgz8Cj4TqmuvtdkdyfZnYVcFX4bgpwCnAJ8CDBvgS4E5hVkzWaWT/gAXcfbGadgReAQmAJcL27F5T1b1pe32qu7xSCi8rnE+ybK9z9h/ClLfsDmeGHnQ8kAq8A9YANwNXuXrlLIR1djT2p5PsiQvvwH0Cr8KYOwFx3H21m/wTSCS4FutfdR9ZUfUcqmkfsFwAp7v4j4HbgoQjVcTmw1d0HAGcDTwC9gYfdfXD4v1fNrBcwCOgHjAb+VBPFmVkKECpRy9XA08ClBG/4fuE3XMT2p7u/UFQfsAD4FcE+vLVE3R/XZI1mdivwLMEHDcDDwPjwv3MIOP8Q/6al+tZAfY8SXGB+MPAmcFu4vTcwosR+3An8BnglXN8i4BdVXV85NR7O+6LG96G7jw7vvx8DO4Cbw127AP3DNRddsLTa6zsa0Rzs/YEpAO4+Fzg1QnW8BkwI3w4RjDp6A6PMbKaZ/cXM0gjqneruhe6+Fkgws+pfmDkY3dY3s6lm9pGZDQSS3X2luxcCHwDDqAX708xOBXq4+zME+/AaM5tlZg+ZWUIN17gS+EmJ+72Bj8O3J7N/n5X1b1pW3+qub7S7fxG+nQBkh7+FdQGeMbPZZnZNeHvxfqzG+sqq8XDeF5HYh0XuAh53941m1hJoDLxjZp+Y2bklXkt113fEojnYGwI7S9zPD7/5a5S773b3zPD/pK8D44H5wK/dfSCwimAa4eB6M4FGNVDiHoIpjRHAdcDz4baD66gN+3McwZsK4EPgRmAg0ICg9hqr0d3fIPjqXSQU/iCE8vdZUXtZfau1PnffCGBmpwM3AI8AqQTTM5cTfJv8pZmddFDd1fb/YRn78HDeFzW+DwHMrAUwlGCaBSCJ4JvhBQQfAo+E+1R7fUcjmoN9F1Dywo5x7p4XiULMrC0wHXjJ3V8B3nL3BeHNbwE9KV1vGsHXveq2HPhbeES0nOBN1LSMOiK6P82sMWDuPj3c9Jy7rwq/ef5J2fuwJmssOX9a3j4rai+rb7Uzs4sJptlGufsWgg/wR919j7tnAh8RfIMrWXeN1cfhvS8isg+BnxJMUxWder0JeNrd89x9M8HUlUWwvkqJ5mCfDZwDYGanAV9FoojwV7WpwG3u/ly4+QMz6xu+PZRg3ng2MMLM4sysHUEoZdRAidcQnos2s2OA+kCWmXUysxDBSH4Wkd+fA4Fp4b8fAhabWdFFcEvuw0jVuMjMBodvj2T/Pivr37SsvtXKzC4nGKkPdvdV4eauwGwziw8f5O8PLKTEfqyp+sIO531R4/swbBjB1ErJ+68BmFkD4ARgaQTrq5SoPSuG4BN/uJl9SjC3fXWE6hgHNAEmmFnRXPv/EHxlyyX4xL/W3XeZ2SxgDsEH6vU1VN9fgBfM7BOCI/jXEIw2Xib4gcNUd59nZp8R2f1pBF/PcfdCM/sv4E0z2wt8A0wiOOMjUjXeAkwysySCN/br7p5fzr9pqb7VWZiZxQOPAWsJ9hnAx+5+p5m9BMwlmHL4q7t/bWb3AC+a2c+BDIID6TXhv4HHK/m+qNF9WELx/4cA7j7ZzEaY2VyC9804d88ws0jVVyn6gZKISIyJ5qkYEREpg4JdRCTGKNhFRGKMgl1EJMYo2EVEYoyCXUQkxijYRURijIJdRCTG/H9fTClmCFzNCwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "he lots I kyour bjt?n, uld ofpyo: lruysAfoV havehe,\n",
      "ve and , Vminne, vay, ' -your Lralns Stry, Bear of's, I bvall.\n",
      "I cew,ew, vadd\n",
      "Le my Vruny:\n",
      "I O spraiJ, bu larrtwes a ' hat! I be the !ue\n",
      "co a parlde\n"
     ]
    }
   ],
   "source": [
    "layers2 = [LSTMLayer(hidden_size=256, output_size=128, weight_scale=0.1),\n",
    "           LSTMLayer(hidden_size=256, output_size=62, weight_scale=0.01)]\n",
    "mod = RNNModel(layers=layers2,\n",
    "               vocab_size=62, sequence_length=25,\n",
    "               loss=SoftmaxCrossEntropy())\n",
    "optim = SGD(lr=0.01, gradient_clipping=True)\n",
    "trainer = RNNTrainer('input.txt', mod, optim, batch_size=32)\n",
    "trainer.train(2000, sample_every=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD0CAYAAACPUQ0CAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3yUReLH8c+mEkpCCRAgNCmDgBRBQCAUQVGxe6fYy1l/lsN29nInWO48z4Jn7yfnWbAioCAghCLSmxMINfQECCEhgST7++PZbDbZQAJsyi7f9+vli93Z2SezG/eb2XnmmXG53W5ERCR0hFV3A0REJLAU7CIiIUbBLiISYhTsIiIhRsEuIhJiIqq7AcaYaOA0YBtQUM3NEREJFuFAM2CBtTbP94FqD3acUJ9V3Y0QEQlSScBs34KaEOzbAD755BMSEhKquy0iIkFh+/btXHXVVeDJUF81IdgLABISEkhMTKzutoiIBBu/IWydPBURCTEKdhGREKNgFxEJMQp2EZEQo2AXEQkxCnYRkRAT1MH+8byNXP/+r9XdDBGRGiWogz1lexZLN++t7maIiNQoQR3sLhdo/ycRkZKCO9gB7ewnIlJScAe7y4X2bBURKSmogx00FCMiUlpQB7vLhZJdRKSU4A52XMp1EZFSgjvYXWiMXUSklOAOdjQSIyJSWnAHu0vTHUVESgvyYHfhVp9dRKSE4A521GMXESktqIMdLSkgIuInqIPdpWQXEfETUZFKxpgmwELgTCAf+AAnUlcAd1hrC40xTwIjPY+Pttb+aoxpX1bdQDXeWQRMyS4i4qvcHrsxJhJ4EzjgKXoReMxam4QzzH2hMeZUYDDQFxgFvHa4uoFsvMbYRUT8VWQo5gXgDWCr534vYKbn9iRgODAQ+NFa67bWbgIijDGND1M3YLRsr4iIvyMGuzHmemCXtXaKT7HLWluUp1lAHBALZPrUKSovq27AhGl1RxERP+WNsd8IuI0xw4EewEdAE5/H6wF7gX2e26XLC8soCxgXUKhcFxEp4Yg9dmvtIGvtYGvtEGAJcC0wyRgzxFPlHGAWkAyMMMaEGWNaAWHW2nRgcRl1A8flCujhRERCQYVmxZRyH/C2MSYKWA18Ya0tMMbMAubi/LG443B1A9Bmr6JYd7vduBTyIiLAUQS7p9deZHAZjz8FPFWqLKWsuoFSlOVutzrvIiJFgv8CJTQzRkTEV3AHu7fHrmgXESkS3MHu+VexLiJSLLiD3WeMXUREHEEe7EVj7Ep2EZEiQR3sRdRjFxEpFtTBrimOIiL+gjvYi6Y7qscuIuIV3MFedPJUY+wiIl7BHeyef9VjFxEpFtzB7u2xi4hIkeAOdu8Yu6JdRKRIcAe7euwiIn6COtiLqMMuIlIsqIPdpS67iIif4A52z7+a7igiUiy4g12LgImI+AnuYPf8q1wXESkW3MHu0nRHEZHSgjrYwzxd9kLluoiIV1AHO1qPXUTET1AHu3fVXuW6iIhXcAe7prGLiPgJ7mDXeuwiIn6CO9i1HruIiJ+I8ioYY8KBtwGDM+pxm+d5bwD5QApwk7W20BhzM3Crp3yMtfZ7Y0w8MB6IAbYCN1hrcwLReK3HLiLiryI99vMBrLUDgMeAscCTwN+stQOBaGCkMSYBuBsYAIwAnjXGRANPAOOttUnAYpzgDwiNsYuI+Cs32K21XwO3eO62BvbiBHRDY4wLqAccAvoAydbaPGttJrAW6AYMBCZ7nj8JGB6oxms9dhERfxUaY7fW5htjPgReBT4B1gCvAKuBpsAMIBbI9HlaFhBXqryoLDC0VoyIiJ8Knzy11l4HdMQZb38FSLLWdgI+Av4J7MPpvReph9O79y0vKgsIV/lVREROOOUGuzHmGmPMw567OUAhsBsnsME5IdoA+BVIMsbUMsbEAScDK4Bk4FxP3XOAWYFqfPFaMYE6oohI8Ct3VgwwAXjfGPMLEAmMBjKAT40x+cBB4GZr7XZjzCs4wR0GPGqtzTXGjAE+9MyYSQeuDFTjtR67iIi/coPdWpsNXFbGQwPKqPs2zlCNb9kO4OxjbeCRaD12ERF/IXKBkoiIFAnuYNd0RxERP8Ed7Oqxi4j4CepgL6IOu4hIsaAO9qLpjuqzi4gUC+5g9/yrHruISLHgDnaNsYuI+AnuYNdGGyIifoI72LXRhoiIn6AO9jBdeSoi4ieog73o9Gmhkl1ExCuog11rxYiI+AvuYK/uBoiI1EDBHexaj11ExE9wB7vnX82KEREpFtzBrjF2ERE/oRHs1dsMEZEaJbiDXeuxi4j4CepgRz12ERE/QR3sWt1RRMRfcAe71mMXEfET3MHu+Vc9dhGRYsEd7BpjFxHxE9TBHuZJ9hVbMvklZVc1t0ZEpGaIqO4GHI9wz7q9f/1uFQAbnhtZnc0REakRgrrHHhGmZcBEREort8dujAkH3gYMznD2bcBOT1kDIBy41lqbaoy5GbgVyAfGWGu/N8bEA+OBGGArcIO1NicgjQ8P6r9LIiKVoiLJeD6AtXYA8BgwFvg78Im1dpCnrJMxJgG4GxgAjACeNcZEA08A4621ScBinOAPiNI99vyCwkAdWkQkaJUb7Nbar4FbPHdbA3txwjvRGDMVuAqYAfQBkq21edbaTGAt0A0YCEz2PH8SMDxQjY8ILxns2XkFgTq0iEjQqtBYhrU23xjzIfAq8AnQBthjrR0ObAIeBGKBTJ+nZQFxpcqLygKidI89K+9QoA4tIhK0KjxIba29DuiIM7a+F/jW89B3QG9gH1DP5yn1PPV8y4vKAiI8rGTz1WMXEalAsBtjrjHGPOy5mwMUAr8A53rKBgErgV+BJGNMLWNMHHAysAJI9ql7DjArUI0v3WPfn5cfqEOLiAStivTYJwA9jTG/AFOA0cC9wLXGmDnA2cAz1trtwCs4wf0z8Ki1NhcYA4wyxiQDpwPjAtX40mPsCnYRkQpMd7TWZgOXlfHQmWXUfRtnqMa3bAdO+AdchN9QjIJdRCSoJ4L7DcXkKthFRII72DUUIyLiJ7iDvdRQjIJdRCTIgz08rPQFSgp2EZGgDnb/C5QU7CIiQR3sYeqxi4j4CepgL02zYkREQi3Y1WMXEQmtYF+1bV91N0FEpNqFVLBn5ebz3dKt1d0MEZFqFTLBHl83GoApK7dXc0tERKpXyAT7dae3BqB2VHjAj/36jFQWbdoT8OOKiFSGkAn2O4a2ByC2VmRAj7st8wDPT/6dGz9YENDjiohUlpAJ9rAwFwmxtdiXG7hdlOavy+D0Z38GYG+OdmcSkeAQMsEOEBsTQeaBwAVw6V76wXxtli0iNV9IBXtcTCT7DhzfXPa5qRm8N3s9KTuyyD5Ycqu96XYnaXtyjuv4IiKVrdyNNoJJXEwkW/fmHtcxrnh73mEfu/XjhQCs+tsIakdV7K3blJHDvtxDdG0RsD28RUSOKOh77M9cfAo3DmgLOCdOV23bR5uHJpJ76Og3ti691sxNA9sy56EzeOfa3iXKe/ztJ/Lyi4//w/JtzE3NKFHH7XaTvDadQf+Yznmvzsbtdh91e0REjkXQ99iv7NvKe7tZ/Vre29syc2kbX6fM5xSF7Kw16fRoVd87k2bCojRvnaVPnkVcjFO+O/tgiecfzC9kbmoGQ0wT7PYs/u+TRQD8/vTZ1IoMJy+/gBd/SuHNmeu8z9mxL4+EuFqIiFS2oA92X2d2TuC16akArN25/7DBftU785nj08O+sm8rujaP4/FvVlI3OoLlT52Fy1W8cmSX5rE8c/EpXNCjOQ9+sYyJy7eRuiubOtG7+eMbc731dmcfJCG2Fl2fnMKhAuePR73oCLLy8pm/PoMLe7SojJctIlJCSAV7l+ax3ttrdmZxZuemfnVyDuaXCHWA8fM3eW8nNogpEeoALpfL+81g3JU9mfjwNp7+fpXfsXfsy2VDRrY31Mff3JfT2jSk65NTeOyrFfx9siXzwCHeuqYX/dvHH/sLFRE5gqAfY/cVGV78cjbvLnv2yqw16QBER4TRNDaakac0K/H4XWd0OOLPcLlcNKwTVaLsk5v6AnDxv+dw04e/ATD7waH0bxdPZHgYNyedRFZePlv2HmB/Xj5XvjP/6F6YiMhRCKlg9zVv3W7vWHphoZtP5m9kf14+oz9dAsCnt/Rj/iPDeeWKnt7nTB6dxMhuzco8nq9p9w7moh7N6dI8lhcv607/do249NREAHIOFnDJqS1IbFDbW//+EcZvqYMXf0rRxiAiUilCLth7tW4AwPr0bMZMXA3Ayq37ePSrFdw1fhEHDhVQKzKMHi3rAyX3TW1Sr2InNxvUieKlUT2ZeHcSl5yaiMvl4p+Xdfc+3qFJPb/nfH7b6Yy/qS9tGjmB/8q0NQx/ceaxvUgRkSMIuWD/9JZ+zH34DABme4ZdijbgmG53AfDoyM4lxtGv79+GBrUjvbNgjtUPdyeR1CGe6/q39nusS/M4+rePZ1Sf4lk82zJzKSzUNEgRCaxyT54aY8KBtwEDuIHbrLUrPI9dCdxlrT3dc/9m4FYgHxhjrf3eGBMPjAdigK3ADdbaSrt8MzI8jGZxMZzZuSmpu/YD+K0f0ymhZI/6yfM78+jIk0v03o9F5+axfPynvkesc9PAthzML2Tl1kymrNzBj6u2c3bX8od/qtvu7IO8Mm0N948w1I0OqXPuIiGnIj328wGstQOAx4CxAMaYnsCfAJfnfgJwNzAAGAE8a4yJBp4Axltrk4DFOMFf6bo0j2V9ejbZefnsK7V+TPvGdUvcd7lcJU68VqaI8DDuHtaBJ8/vAsDH8zZWyc8tze12M2dteoUv5Hr15zV8MGcD32sjE5Ear9w0s9Z+Ddziudsa2GuMaQQ8A4z2qdoHSLbW5llrM4G1QDdgIDDZU2cSMDxAbT+iLs3jcLuh599+4sdVOwCY/8gwZv1lKA1KzWqpDs3rx9ApoR7JazMOO4OnMo37eS1XvjO/xFTPIynaKHz5lszKbJaIBECFuqnW2nxjzIfAqzjDKu8C9wJZPtViAd9PfRYQV6q8qKzSFc1pP1hQyE+rdlArMoz4utG0bFi7nGdWnecv7QbAsrTAhKXb7a7w0gUrtzr7w9rtWeXUdBRtNDJl5Q6dFxCp4So8/mCtvQ7oCHwNdAdeBz4FOhtjXgL2Ab6D1/WAvaXKi8oqXbNSl++3alj7uMfQA61Ts3qEueDHVce/nV/uoQI6PjaJq9+t2Bz59P15AKze7r8B+ObdOTw36XcmLtvGhvRstu49QOqubJrGRpO+P49t+45voTURqVzlBrsx5hpjzMOeuznAdqCztXYIMApYZa0dDfwKJBljahlj4oCTgRVAMnCu5/nnALMC+xLK5nK5iPIZN0/Zsb8qfuxRiY4Ip1lcDN8s2cqmjBx2ZuXyk2fY6Ghd/tY8DhW4SV6bwXS784h1Cwvd/LbR6YEvS8tkb07JtXBu+89C3piZyh3jFzHkhRn0f87ZbGT08I4ArNu1n4/nbmDeugyue+9X2jw0UXvNitQgFemxTwB6GmN+AaYAo621B0pXstZuB17BCe6fgUettbnAGGCUMSYZOB0YF6jGl+f87s29t4v2RK1pnrv0FADmrkvnwS+WcfNHvx11uM9bl8HSzcVfhG54fwHz12WUWXdZ2l5OeuQHAOrVcma39PjbTzz61XIKPEMs2zL9e+RPnt+ZMzo1AeCz39J4/JuVjHprHjNTnCmkt368kAtfSyZlR8WGdkSk8riqezlZY0wbYP20adNITEwM6LHz8gtYvS2Ldo3rEBMZTkQVzXw5GrmHCuj0+OQSZXed0Z77zjIVPsbF/05m8aa9jLmoKws37uGrxVsA+PbOAXy5MI37RxjqeVawHP7iTNbu3E9EmIsVfx1R4mdPGT2INvG16fzEFAZ3bMzQTk3I2J/Hpt05/P3SboSHuWj78A9HbMvV/Vox5qJTvPdXb9vHS1NTePGyHtTRNEmRgElLS2PYsGEAba21G3wfC+lPWnREuPcK05qqVmQ4gzs29vZ8AVZvq3ivd966DBZvcnrrF/ZozpV9WhEZ7uKz39K4YFwyADFRETx0TicOFRSyIT2bS09N5IU/dsPlcvGHXol8sdBZrviu/y4iO6+AgkI3157emiGmid/PO69bM75ftq1E2ZyHzmDBht38+dMlLNy4l617DxAdEca2zFwufX0OefmFzEnNYFDHeKIjwv2OKSKBFdLBHiw+vLEPm3fnkNgghvs+X+q9YtZXQaGb9P15NI0teVL4fws2A/DYyJO9vfLbh7Tns9+K15Z/Y2Yq53VrRp3oCPIL3ZzerpH3ytvr+7dhfXo2K7Zkes9DJHWIZ1CHxmW2ddyVp/Kvywt5c2YqL/yYwpe3n07z+jFc2KMFKTuyeG16qndM3tfNH/1Gs7hajLuyJ4cK3PQ7qdExvFMiUhE1b2ziBNWyYW1cLhemaT12ZuWxx2dzD7fbzQXjZtP3mWnc//lS76ba6fvz+GrxFhrXi+ampJO89dvG12HDcyOZ+cAQ3r/hNADembWOjRnZAN71agC6tojjy9v706dtQ2/Z9f3bEHaEGUSR4WHceUYH1j97Lr1aFz/vnuEd6dOmoV/9c7omAHh68HMZ9dY87SglUokU7DVM0d6oQ16YwYdzNgCwISPHO+/8i4VpLPbMKZ/oGRLp29Y/TAFaN6rDUNOEs7sk8PWSrVz//gIAWjXyn8v/1AVdGNSxMTcOaMvgjmX31ksrvW59RHgYn912Or7FZ3ZuyutX92LJE2eWqPv4Nyv4JWWXAl6kEmgopobp7jknkHngEE9+u5INGdl+q05e/tY8pt47iA2eHvgLf+zudxxfF/VswWSf6YiN60b71WnXuC4f3djneJsPwGe3ns4vKbu4KekkIsOdlK9fO4oRXZoyZaUz4+c/8zbxn3mb6N+uEeNv7gfAzqxc+j0zjQ9u6MOgCv5xERF/6rHXMHWjI7ikZ/EWeu8nb+D5yb8TXzea2Q8O9ZZ/PHcjSzfvpXtiHLUij3xC8nSf8eyXLu/h19MOtNPaNOS+swxxMZHUjiruO7x+VS82PDeS7+8a6C2bk5rh7bX/tGoHhW649r1fj2kzchFxKNhroH/8sTuzHxzK13cM8JaNHt6BxAa1mfPQGZim9fhw7kYWbdpLq0Zl7+vqK652JP1OcoZrWjaMqbR2l6do3L5L81geGGG8q2w+/s0KdmXl8ZnnRDBAp8cns9BzEZWIHJ2QnsceKnZl5RFfN8rb007bk8PA56cD8MAIwx1D25d7jAMHC5j2+w7O7drsiCdGq9Kqrfs495WSFyLfMbQdn/+Wxs6sPLo0j2Xi3UnV1DqRmu1I89jVYw8CjetFlxg+SWxQ2zukcdvgdhU6RkxUOOd1a15jQh2gfZO6nFVqw/FOCbHMf2QY4CxU9sq0NdXRNOasTefd2eu9V+OKBBMFexDr2iKuxi1sdjSiIsJ469re3j9Ofds25NxTmuFyubwncl/8KaXctW8CraDQzfXvL+Dp71eRvNb/mgKRmk7BLtXuoXM6sfCx4Xx6Sz/vH6pBHRtzUQ9nrZ8b3l9QpdMi0/bkcLDAuVZA4/wSjBTsUiM0qhvtN1tn7MWn0NuzOfm/Z6RWWVsmrSieGvrytDVaf16CjoJdaqw60RGMv7kfjepE8Y8plrU7q2blyI0Zzo5Wl/V2TuYXXS8gEiwU7FKjRUWE8eoVPQH4domz32pBoZu8fGexMrfbzQfJ63l20mrv5iHHY8WWTL5bupXerRtwZV9nqec1O2veWv4iR6IrT6XG698+nhb1Y1i1bR8XjpvNUs9Wgqe1acADIzrx1HerAJi2eidT7x18TD/D7Xbz7xmp/GOKBWBA+3g6NKlLVEQYc9amM6JLQmBejEgVUI9dgsKwk5swdfVOb6gDLNiwh00+G4Gv3bmfX9fv9nvu5t05jP50MevTsxk/f5P3RKzb7eau/y7mtelrmb023RvqADcOaEud6AhM03qsz6j6zcZFjoeCXYLCH3oVX7zWrnEd+rdzlkkoWsf+6Yu6AnDZm3P95p5PWLSFr5dsZegLM3jkq+V8ucjZiCRtzwG+W7qVf0yx3kXWWjeqzYz7hxBX21kCuWXDGH5J2cX2MnaVEqmpFOwSFLol1mfN2HOY/eBQpt03hBcv6wHAd0udcfer+7by1i29Pd+c1JJz0V+amgLAlr3FOzw+N+l3WtSPYeYDQ2kTX7xMQ94hZ9rjte9VbJNwkZpAwS5BIzI8jMQGzpLDCXG1uNazj+2FPZrjcrm46wxnaYUvFxZvMvJLyi7mr99Ni/rFa+Sk7TnA5t05LCg1bHNet2Z+P/PGgW0BZzP0Ng9N5I7xi/hswWbem72e3zb4D/uI1ARaK0ZCyrB/ziB1VzbT7htMu8Z1eejLZXy6YDOLHj+TnVm5vD4jlW+WbCUuJpL9efneYZuOTevy7Z0Dy1wp8/UZqTw/+Xe/8oTYWszzLH8gUtW0VoycMJ66oAsAl785lw3p2XzqWTGyYZ0oOiXE8vKontw6+CQyDxyioNDNO9f2ZsNzI/nxnsGHXf74vG7NOLuMWTGHPFenitQ0CnYJKUkdGjOgfSPS9x9kyAszALj/rI4l6vx5WIfi+h3jyz1my4a1eeOaXvz+9Nn0at2AMRd15b4zO5KRfZDMnEMBbb9IICjYJeQ8NrKz9/aprepz5xkdSjzuu/lHdMSRNynxVSsynC9v78/V/Vp7tzBcU0VXw4ocDV2gJCGnVcPiPV2fueSUMuv884/daRZXq8zHKqJ9k7qAc1K1dxkbeItUJwW7hJw60RHcd2ZHDhYU0ikhtsw6l/Y6vhP1LerHEB7m4pGvlnNxzxbERFW85y9S2RTsEpLuGtah/ErHISzMRWytCPbkHOKz3zZzXf82lfrzRI5GucFujAkH3gYM4AZu8zzvVaAAyAOutdbuMMbcDNwK5ANjrLXfG2PigfFADLAVuMFaq2u0Jei9c91pXPr6HN6cmcr53ZvTsE5UdTdJBKjYydPzAay1A4DHgLHAy8Bd1tohwATgQWNMAnA3MAAYATxrjIkGngDGW2uTgMU4wS8S9Hp51orfmpnLIxOWV3NrRIqVG+zW2q+BWzx3WwN7gVHW2iWesgggF+gDJFtr86y1mcBaoBswEJjsqTsJGB645otUryGmMQCTV25nkxYLkxqiQtMdrbX5xpgPcYZfPrHWbgMwxvQH7gT+BcQCmT5PywLiSpUXlYmEhFev6Ekdz4nT6z/4tZpbI+Ko8Dx2a+11QEfgbWNMHWPM5cAbwEhr7S5gH1DP5yn1cHr3vuVFZSIhoV6tSD76k7Px9v7c/GpujYij3GA3xlxjjHnYczcHKAQuwempD7HWrvM89iuQZIypZYyJA04GVgDJwLmeOucAswLYfpFq16t1Q24b3I70/XkczNcyA1L9KtJjnwD0NMb8AkwBRuOcPK0HTDDGzDDG/NVaux14BSe4fwYetdbmAmOAUcaYZOB0YFwlvA6RatWucR0K3fDO7HXlVxapZOVOd7TWZgOXlSou81I7a+3bOFMjfct2AGcfawNFgsFZXRJ44ItlTF21g/8b0r66myMnOK0VIxIAcTGRDO7YmPzC6l0GWwQU7CIBk9gghg3p2cxM2cXCjdqEQ6qPgl0kQAa2j2dfbj7Xvfcrl74+t7qbIycwBbtIgJzZuWmlHDfnYD6Xvj6HC19L1vrvUiEKdpEAiQgPo13j4o2wd+zL5cDBAv63YBP5BYWsT89m6AszmL8u47DH2JZ5gI0Z2QC43W7uGL+I5yb9zsKNe1i6eS836CIoqQCt7igSQO9f34fk1HQenrCc37dnsWD9bsZNX8vB/EKWbM5kfXo2k1Zsp32TutzwwQLO69aMWwa18z7/lo8WsnxLJj/cncS01TuYuGxbiePnHCzw3i4odPPA50tJbBDDvWeZKnuNUvMp2EUCqFWj2sRENeVhlvPZb5uJCHMB8Pg3K711PpizgeS16azZuZ9laZk888Pv/HB3EvmFhSzf4qy+ce4rJa/j69+uEfmFbgoK3WzMyGbcz2tZsXUfq7ftAygz2H9J2cWBQwWMKGO/VgltCnaRAIuv6yzfW9Tbjgx3wv1QQfFUyDU795d4TukgL+2afq35adUOJizewuB/zPB7PG1PDokNajMnNZ2+bRuxZPNern3PGbb57bHhxNeNPubXI8FHY+wiAeZyufj+roHe+3cO7cDyp0bQLTGOT27qS+dmzq5ORYFf2v9u6cffL+3G0ifPYskTZ/Lg2Z044+QmtPTZ8q9Iv5OcawWvfmc+yWvTufLt+fzrpxQufX2Ot86tHy8kv0BLHZxIXG539V5QYYxpA6yfNm0aiYnHt12ZSE2Sl1/AnLUZJHWIJyK8uA9VWOjmveT19GnbkGVpmWzek8NprRsyJzWDnq3qc3735mUeb1naXi4Yl1yibOq9g7lw3GyyDxaQ2CCGtD0HaBtfh8hwFyk7ir8V/PfmfpzerlHlvFCpFmlpaQwbNgygrbV2g+9jGooRqSTREeEM7dTErzwszMVNSScB0C2xvrd8eDnTJbsl1mfuw2eQEOtswp225wAtG9bmv7f044JxyaTtOQDA+nRnVk3ftg2Zv965UGrHvlwA8gsKS/yRkdCk37BIEGkWF4PL5cLlcnmHZrol1qd7orPNQVKHeG/dwaYx0+4bDMDGjBw2786h85NTeGeWFioLdeqxi4SAN6/pzcqtmQxoH8+kFc5J24t6tMDlctGqYW1enpZC8/q1OJhfyJiJq6lXK4LLT2tVza2WyqJgFwkBCXG1SIhzhmgu7lnyXFX3lvXZtDuHB75Y5i178MvlXNijBbUiw6u0nVI1NBQjEuLuGNquxP2xF3cF4OO5G6nKyROFhW5em76WlB1ZR/WcnIPamepoqccuEuI6JcTy7CWnMDc1gz8P78C+A856M2N/WE2T2Ggu7NGC/IJCvlu2lXO6Nqu0XvzyLZn8Y4plyea9vH1t7yPWXbp5Lxe+VjwDqGXDGH55YCgul4tDBYWs3bmfkz3TRsWfeuwiJ4Ar+rTilSt60q5xXU6Kr+stHzNxNfvz8mn/6CTu+d9S7vnfkkprw9I0Z7vjn1btYMWWTCav2F5mvX25h0qEOntRr6QAAA6ySURBVMDm3Qd4eMJy3G43nZ+YzDkvz8Jur3jP/0SjYBc5wcTVjuTL2/szxDRmV1YeExaleR9L3bWfgkrYLMTtdvOEz7IK5706m9v+s5D16dl+w0FrfObfX9GnFQ+e3QmATxdspu3DP3iv4B37w2oO+KydI8UU7CInoF6tGzB6eEcAb+D2b9eIlB37affID3y2YHNAf97KrfvKLB/6wgyGvTizxCbgz0/6HYDp9w/h2UtO4dZBJ9GjZf0Sz0uIrcUvKbs4+YnJHCoo5KWpKbR5aCK7svIC2u5gpWAXOUGd3Kyed10bcHrHRf7y5TIKA9hz/8+8jQB8eGMfv8fW7crmirfn4Xa7eWlqCr9ucC6qahvvLIEcFubi6zsG0KJ+DABndGrCuCt7ep/f4dFJvDR1DQCnjZ1apSeEayoFu8gJKjoinOcv7UZUeBiT/pxE35NK7lE/M2VXucfILyjkmyVbWJ7mrEpZFKprdmQxZaUzhr435yCfer4BJLWP54nzOnN9/zYseHQ473hOoi7cuIfJK7Z7A/q7OweW/lFMHp3EH3olcs/wjvRu05B5Dw8rs02H+3ZwItFaMSLideMHC2gbX4cP52zg3FOa0e+kRjSsE8Xgjo2JifKfLXPNu/OZtSYdgPvP6sinCzbTtXkckz2hvv7Zc/nX1DW8Mm0Ndw5tz/0j/JcXnr0mnavfne+9f3NSWx4d2bnCbT77pV9I23OAL2/vz4iXfgEgZcw5REWEdr9Va8WISIW8d/1pAPz8+06+XbqVb5duBaBhnShmPziU2lElI6Mo1AFe+DEFwLtmDTjr1nzx22Y6N4tl9PAOZf5M38XJ6teOPKpQB/juroFk5+VTv3bxsNK3S7fyh14nbkcxtP+kicgxKVpIrMju7IPc9OFv3vsb0rNZuHEPAJf1TqRxvbLXez/jnzPZmpnLVf1aHXbxsfAwl3etm1dG9SyzzpFEhod5Q33qvYMAZyjoRKYeu4j46ZRQj98988TP7pLA5JXbmZOawaJNe2hUJ4ohL8zw1h3VpxV//0N3NmXksGZnFhOXbePy01py+VvzAGfjkbPL2cXp/Rv6sHjTHgZ1bHxc7W7fpB7dEuP4clEa951lQn445nAU7CLiZ/zN/dibc5A2jeoQFuZiTqqziccl/55DN0/vukg7zwVPrRrVplWj2gw7uSn5BYVc378Nl56aSJfmsYSFlb2pSJGGdaIYdvKRly2uqD8NbMufP13C8i2Z9GrdICDHDDblBrsxJhx4GzCAG7gNyAU+8NxfAdxhrS00xjwJjATygdHW2l+NMe3Lqhv4lyIigdKwThQN6xSPWZ9+UiMa1YkiI/sgy9IyObVVfVJ27OfmpJOIqx3p9/yI8DCeuqBLVTbZ69RWTpjb7VknbLBX5HvK+QDW2gHAY8BY4EXgMWttEuACLjTGnAoMBvoCo4DXPM/3qxvQVyAilc7lcnHvWR299y/r3ZIVfx3Bnw9zQrQ6JTaIoW50BL955sOfiMoNdmvt18Atnrutgb1AL2Cmp2wSMBwYCPxorXVbazcBEcaYxoepKyJBxvfqz7O7HnnMvDq5XC4a1Y1iwuIt7PTsHHWiqdCZBWttvjHmQ+BV4BPAZa0tmgCfBcQBsUCmz9OKysuqKyJBpnOzWMZd2ZMFjw4vMbWwJrrrDOebxHmvziYv/8RbT6bCp4yttdcBHXHG22N8HqqH04vf57ldurywjDIRCTIul4vzujU/7NTGmqRoDvvOrDzGTlwdsOPu2JcbFEsWlBvsxphrjDEPe+7m4AT1b8aYIZ6yc4BZQDIwwhgTZoxpBYRZa9OBxWXUFRGpVBP+rz8AH83dyKaMnOM+3rbMA/R9Zhp3jl983MeqbBXpsU8AehpjfgGmAKOBO4C/GmPmAlHAF9bahTihPRf40lMH4L7SdQP7EkRE/J3aqgFNPN8uhv9rJmt37i/nGUf231+d9W4mLt/GjyvLXku+ptBaMSISsg4cLODkJyYDkNQhno//1PeYj3Xa2KneZYGv6tuKsRef4ldnZsouGtSOpFti8Ynmaat3MPaH1Xx+6+k0qhu4YawjrRVzYl6WJSInhJiocO73TNOctSa9zBOpm3fnlLtEcWGhu8Ra78vSMv3qfJC8nuve+5WLfHZ/ytifx58+/I11u7KZsnLHsb6Mo6ZgF5GQducZHXjhj90BePr7VQCM+3kNSX//mX9M+Z2kv0/3rgp5ON8tcxZDG3txVy7o3pzlWzLp9tQU2jw0kd+372PL3gM89Z1z7EI3TFq+jUWb9tBrzFTvMR75arl3DZs3Z6bS5qGJ3PvZEnIPBX7WjpYUEJGQd373Ztz/+VKWbs5kWdpe70qUr01PBWDNzv3syz1EbC3/q2gLCt38tGoHCbG1uOK0VrRtVIdvl25lX24+AF8uTGPT7pInZ2//ZFGJ++eeksAPy7fzxsx1LNq0x7vI2oRFW7hneEdaNqwd0NerHruIhLzoiHBuG9yO5VsyuWCcM1TSwLMUQutGTqj2fnoqy9JKzsbu9tQU2j3yA1NX7+DU1vUJC3PRv308keHFa9+8PWs9U1buYED7Rmx4biSz/jIU07R45vcbV/fi739wvjF8uSitxMqZ/77q1ICHOqjHLiIniGEnN+GNmU4PvUX9GKbfP4Rx09dy+Wktufbd+aTuyuaCcclseG4kBYVu2j3yg/e5uYcK6dmyeN2Zd687jW+XbiW/oJCvlzjDNHd7Lopq2bA2U+4ZhNvtJvdQoXeDkqJVMgGu6deapy/qWmmvVcEuIieE3j4Lgr121alERYRx75nOidUPb+zDwOenA3D/50vJy/dfp3CIKV5SeFDHxt4lhi/q2YL8Ajd9T2pUor7L5Sqx69S4K3vS/tFJAPyxd+XOAFSwi8gJweVycc/wjsTGRJRY9wYgsUFtFjw6nNPGTuWLhWne8t+fPptCt5u0PQfo0LRe6UMCMMQ0qdDPjwgPY+3Yc8jIPkjT2FrH/kIq8rMq9egiIjXIkVajbFwvmqjwMA4WOL31L2/vT61Ip8fd8TChfrQiwsMqPdRBwS4i4rX0ybNwuSAqPKzczUFqMgW7iIiH75h4MNN0RxGREKNgFxEJMQp2EZEQo2AXEQkxCnYRkRCjYBcRCTE1YbpjOMD27TV7RxIRkZrEJzP95mjWhGBvBnDVVVdVdztERIJRMyDVt6AmBPsCIAnYBgR+xXkRkdAUjhPqC0o/UO17noqISGDp5KmISIipCUMxx8QYEwb8G+gO5AE3WWvXVkM7IoH3gDZANDAG2Ax8D6zxVHvdWvs/Y8yTwEggHxhtrf21itq4CNjnubseeBN42dOOH621f63O99MYcz1wveduLaAHcAXwAs57CfAkMKsq22iM6Qs8b60dYoxpD3wAuIEVwB3W2sKyfqeHq1vJ7esBvIoznJkHXGut3WGMeRkYCGR5nnYhEAmMB2KArcAN1tocvx8Q+Db2pIKfi2p6Dz8FEjwPtQHmWWtHGWO+AeKBQ8ABa+05VdW+YxXMPfaLgFrW2tOBh4B/VlM7rgYyrLVJwNnAOKAX8KK1dojnv/8ZY04FBgN9gVHAa1XROGNMLcDl05YbgDeAK3E+8H09H7hqez+ttR8UtQ9YCNyN8x7+xafdM6uyjcaYvwDv4PyhAXgReMzze3YBFx7hd+pXtwra9zJwl+c9nAA86CnvBYzweR8zgSeA8Z72LQZuDXT7DtPGo/lcVPl7aK0d5Xn/Lgb2Avd4qnYABnrafE5Vte94BHOwDwQmA1hr5wG9q6kdnwOPe267cHodvYCRxphfjDHvGmPq4bT3R2ut21q7CYgwxmdLlsrTHahtjPnRGPOzMWYQEG2tTbXWuoEpwHBqwPtpjOkNdLHWvoXzHt5ojJlljPmnMSaiituYClzic78XMNNzexLF71lZv9Oy6lZ2+0ZZa5d4bkcAuZ5vYR2At4wxycaYGz2Pe9/HSmxfWW08ms9FdbyHRf4KvGqt3WaMaQrUB74zxsw2xpzn81oqu33HLJiDPRbI9Llf4PnwVylr7X5rbZbnf9IvgMeAX4EHrLWDgHU4wwil25sFxFVBE3NwhjRGALcB73vKSrejJryfj+B8qAB+Au4CBgF1cdpeZW201n6J89W7iMvzhxAO/54VlZdVt1LbZ63dBmCM6Q/cCfwLqIMzPHM1zrfJ/zPGdCvV7kr7/7CM9/BoPhdV/h4CGGOaAMNwhlkAonC+GV6E80fgX546ld6+4xHMwb4P8N3WJMxam18dDTHGtASmAx9ba8cDX1lrF3oe/groiX976+F83atsKcB/PD2iFJwPUcMy2lGt76cxpj5grLXTPUXvWWvXeT4831D2e1iVbfQdPz3ce1ZUXlbdSmeMuRxnmG2ktXYXzh/wl621OdbaLOBnnG9wvu2usvZxdJ+LankPgT/gDFMVTb3eDrxhrc231u7EGboy1di+CgnmYE8GzgUwxvQDlldHIzxf1X4EHrTWvucpnmKM6eO5PQxn3DgZGGGMCTPGtMIJpfQqaOKNeMaijTHNgdpAtjGmnTHGhdOTn0X1v5+DgGmen+8Clhljinb89X0Pq6uNi40xQzy3z6H4PSvrd1pW3UpljLkap6c+xFq7zlPcEUg2xoR7TvIPBBbh8z5WVfs8juZzUeXvocdwnKEV3/ufAxhj6gJdgdXV2L4KCdpZMTh/8c80xszBGdu+oZra8QjQAHjcGFM01n4vzle2Qzh/8W+x1u4zxswC5uL8Qb2jitr3LvCBMWY2zhn8G3F6G5/gXODwo7V2vjFmAdX7fhqcr+dYa93GmJuACcaYA8Aq4G2cGR/V1cb7gLeNMVE4H+wvrLUFh/md+tWtzIYZY8KBV4BNOO8ZwExr7ZPGmI+BeThDDh9Za1caY8YAHxpjbgbScU6kV4XbgVcr+Lmo0vfQh/f/QwBr7SRjzAhjzDycz80j1tp0Y0x1ta9CdIGSiEiICeahGBERKYOCXUQkxCjYRURCjIJdRCTEKNhFREKMgl1EJMQo2EVEQoyCXUQkxPw/GOCL31dHWuQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " bgr,\n",
      "th crt woerie, mlus no badese t wivete rune soemly,\n",
      "T\n",
      "O, t:\n",
      "woedaw,\n",
      "orrf,e br yO euatge se wanm dinye . tinn wTeiles ne toe nd wigt\n",
      "inP\n",
      "cth ihe fos bish se wer arsethy serfsibrmt pre !omey we re\n"
     ]
    }
   ],
   "source": [
    "layers3 = [GRULayer(hidden_size=256, output_size=128, weight_scale=0.1),\n",
    "           LSTMLayer(hidden_size=256, output_size=62, weight_scale=0.01)]\n",
    "mod = RNNModel(layers=layers3,\n",
    "               vocab_size=62, sequence_length=25,\n",
    "               loss=SoftmaxCrossEntropy())\n",
    "optim = AdaGrad(lr=0.01, gradient_clipping=True)\n",
    "trainer = RNNTrainer('input.txt', mod, optim, batch_size=32)\n",
    "trainer.train(2000, sample_every=100)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
